diff --git a/Directory.Build.props b/Directory.Build.props index bda097b1cf..e784849eed 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,7 +2,7 @@ 6.9.1 2.120.0 - 4.3.0-main-9408418 + 4.3.0-agr-gal-stsdk-9768098 diff --git a/src/NuGetGallery.Core/Auditing/CloudAuditingService.cs b/src/NuGetGallery.Core/Auditing/CloudAuditingService.cs index 64b31eba87..486b09c066 100644 --- a/src/NuGetGallery.Core/Auditing/CloudAuditingService.cs +++ b/src/NuGetGallery.Core/Auditing/CloudAuditingService.cs @@ -5,9 +5,6 @@ using System.Globalization; using System.IO; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; -using Microsoft.WindowsAzure.Storage.Blob.Protocol; using Newtonsoft.Json; using NuGetGallery.Auditing.Obfuscation; @@ -58,23 +55,16 @@ protected override async Task SaveAuditRecordAsync(string auditData, string reso { await WriteBlob(auditData, fullPath, blob); } - catch (StorageException ex) + catch (CloudBlobContainerNotFoundException) { - if (ex.RequestInformation?.ExtendedErrorInformation?.ErrorCode == BlobErrorCodeStrings.ContainerNotFound) - { - retry = true; - } - else - { - throw; - } + retry = true; } if (retry) { // Create the container and try again, // this time we let exceptions bubble out - await container.CreateIfNotExistAsync(permissions: null); + await container.CreateIfNotExistAsync(enablePublicAccess: false); await WriteBlob(auditData, fullPath, blob); } } @@ -89,29 +79,25 @@ private static async Task WriteBlob(string auditData, string fullPath, ISimpleCl { try { - using (var stream = await blob.OpenWriteAsync(AccessCondition.GenerateIfNoneMatchCondition("*"))) + using (var stream = await blob.OpenWriteAsync(AccessConditionWrapper.GenerateIfNoneMatchCondition("*"))) using (var writer = new StreamWriter(stream)) { await writer.WriteAsync(auditData); } } - catch (StorageException ex) + catch (CloudBlobConflictException ex) { - if (ex.RequestInformation != null && ex.RequestInformation.HttpStatusCode == 409) - { - // Blob already existed! - throw new InvalidOperationException(String.Format( - CultureInfo.CurrentCulture, - CoreStrings.CloudAuditingService_DuplicateAuditRecord, - fullPath), ex); - } - throw; + // Blob already existed! + throw new InvalidOperationException(String.Format( + CultureInfo.CurrentCulture, + CoreStrings.CloudAuditingService_DuplicateAuditRecord, + fullPath), ex.InnerException); } } - public Task IsAvailableAsync(BlobRequestOptions options, OperationContext operationContext) + public Task IsAvailableAsync(CloudBlobLocationMode? locationMode) { - return _auditContainerFactory().ExistsAsync(options, operationContext); + return _auditContainerFactory().ExistsAsync(locationMode); } public override string RenderAuditEntry(AuditEntry entry) diff --git a/src/NuGetGallery.Core/Extensions/StorageExceptionExtensions.cs b/src/NuGetGallery.Core/Extensions/StorageExceptionExtensions.cs deleted file mode 100644 index 8260c55d25..0000000000 --- a/src/NuGetGallery.Core/Extensions/StorageExceptionExtensions.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Net; -using Microsoft.WindowsAzure.Storage; - -namespace NuGetGallery -{ - public static class StorageExceptionExtensions - { - public static bool IsFileAlreadyExistsException(this StorageException e) - { - return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.Conflict; - } - - public static bool IsPreconditionFailedException(this StorageException e) - { - return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.PreconditionFailed; - } - } -} diff --git a/src/NuGetGallery.Core/Features/EditableFeatureFlagFileStorageService.cs b/src/NuGetGallery.Core/Features/EditableFeatureFlagFileStorageService.cs index 3704e4e35c..a16e046a5e 100644 --- a/src/NuGetGallery.Core/Features/EditableFeatureFlagFileStorageService.cs +++ b/src/NuGetGallery.Core/Features/EditableFeatureFlagFileStorageService.cs @@ -6,7 +6,6 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; using NuGet.Services.Entities; using NuGet.Services.FeatureFlags; @@ -117,7 +116,7 @@ private async Task TrySaveInternalAsync(FeatureFlags flags, s return ContentSaveResult.Ok; } } - catch (StorageException e) when (e.IsPreconditionFailedException()) + catch (CloudBlobPreconditionFailedException) { return ContentSaveResult.Conflict; } diff --git a/src/NuGetGallery.Core/ICloudStorageStatusDependency.cs b/src/NuGetGallery.Core/ICloudStorageStatusDependency.cs index 0bcf42b422..d7d345509e 100644 --- a/src/NuGetGallery.Core/ICloudStorageStatusDependency.cs +++ b/src/NuGetGallery.Core/ICloudStorageStatusDependency.cs @@ -2,8 +2,6 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; namespace NuGetGallery { @@ -13,6 +11,6 @@ namespace NuGetGallery /// public interface ICloudStorageStatusDependency { - Task IsAvailableAsync(BlobRequestOptions options, OperationContext operationContext); + Task IsAvailableAsync(CloudBlobLocationMode? locationMode); } } \ No newline at end of file diff --git a/src/NuGetGallery.Core/Login/EditableLoginConfigurationFileStorageService.cs b/src/NuGetGallery.Core/Login/EditableLoginConfigurationFileStorageService.cs index 08509e6a98..bbc1d90c54 100644 --- a/src/NuGetGallery.Core/Login/EditableLoginConfigurationFileStorageService.cs +++ b/src/NuGetGallery.Core/Login/EditableLoginConfigurationFileStorageService.cs @@ -5,10 +5,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Runtime.InteropServices.ComTypes; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; using NuGetGallery.Shared; @@ -123,7 +121,7 @@ private async Task TrySaveInternalAsync(LoginDiscontinuation return ContentSaveResult.Ok; } } - catch (StorageException e) when (e.IsPreconditionFailedException()) + catch (CloudBlobPreconditionFailedException) { return ContentSaveResult.Conflict; } diff --git a/src/NuGetGallery.Core/NuGetGallery.Core.csproj b/src/NuGetGallery.Core/NuGetGallery.Core.csproj index 221a3609d7..0858459fe1 100644 --- a/src/NuGetGallery.Core/NuGetGallery.Core.csproj +++ b/src/NuGetGallery.Core/NuGetGallery.Core.csproj @@ -43,6 +43,11 @@ + + + + + $(NuGetClientPackageVersion) @@ -50,9 +55,6 @@ $(ServerCommonPackageVersion) - - 9.3.3 - diff --git a/src/NuGetGallery.Core/Services/AccessConditionWrapper.cs b/src/NuGetGallery.Core/Services/AccessConditionWrapper.cs index 2350910df0..fbbf1bff4e 100644 --- a/src/NuGetGallery.Core/Services/AccessConditionWrapper.cs +++ b/src/NuGetGallery.Core/Services/AccessConditionWrapper.cs @@ -1,8 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using Microsoft.WindowsAzure.Storage; - namespace NuGetGallery { public class AccessConditionWrapper : IAccessCondition @@ -28,20 +26,20 @@ public static IAccessCondition GenerateIfMatchCondition(string etag) { return new AccessConditionWrapper( ifNoneMatchETag: null, - ifMatchETag: AccessCondition.GenerateIfMatchCondition(etag).IfMatchETag); + ifMatchETag: etag); } public static IAccessCondition GenerateIfNoneMatchCondition(string etag) { return new AccessConditionWrapper( - ifNoneMatchETag: AccessCondition.GenerateIfNoneMatchCondition(etag).IfNoneMatchETag, + ifNoneMatchETag: etag, ifMatchETag: null); } public static IAccessCondition GenerateIfNotExistsCondition() { return new AccessConditionWrapper( - ifNoneMatchETag: AccessCondition.GenerateIfNotExistsCondition().IfNoneMatchETag, + ifNoneMatchETag: "*", ifMatchETag: null); } } diff --git a/src/NuGetGallery.Core/Services/BlobListContinuationToken.cs b/src/NuGetGallery.Core/Services/BlobListContinuationToken.cs new file mode 100644 index 0000000000..4b7a02534b --- /dev/null +++ b/src/NuGetGallery.Core/Services/BlobListContinuationToken.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGetGallery +{ + public class BlobListContinuationToken + { + internal BlobListContinuationToken(string continuationToken) + { + ContinuationToken = continuationToken; + } + + internal string ContinuationToken { get; } + } +} diff --git a/src/NuGetGallery.Core/Services/BlobResultSegmentWrapper.cs b/src/NuGetGallery.Core/Services/BlobResultSegmentWrapper.cs index c998f2c265..381c6dd3b8 100644 --- a/src/NuGetGallery.Core/Services/BlobResultSegmentWrapper.cs +++ b/src/NuGetGallery.Core/Services/BlobResultSegmentWrapper.cs @@ -2,23 +2,21 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Collections.Generic; -using System.Linq; -using Microsoft.WindowsAzure.Storage.Blob; namespace NuGetGallery { public class BlobResultSegmentWrapper : ISimpleBlobResultSegment { - public BlobResultSegmentWrapper(BlobResultSegment segment) + public BlobResultSegmentWrapper(IReadOnlyList items, string continuationToken) { // For now, assume all of the blobs are block blobs. This library's storage abstraction only allows // creation of block blobs so it's good enough for now. If another caller created a non-block blob, this // cast will fail at runtime. - Results = segment.Results.Cast().Select(x => new CloudBlobWrapper(x)).ToList(); - ContinuationToken = segment.ContinuationToken; + Results = items; + ContinuationToken = new BlobListContinuationToken(continuationToken); } public IReadOnlyList Results { get; } - public BlobContinuationToken ContinuationToken { get; } + public BlobListContinuationToken ContinuationToken { get; } } } diff --git a/src/NuGetGallery.Core/Services/CloudBlobClientWrapper.cs b/src/NuGetGallery.Core/Services/CloudBlobClientWrapper.cs index 5dd42356d5..3df90023da 100644 --- a/src/NuGetGallery.Core/Services/CloudBlobClientWrapper.cs +++ b/src/NuGetGallery.Core/Services/CloudBlobClientWrapper.cs @@ -2,30 +2,51 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Auth; -using Microsoft.WindowsAzure.Storage.Blob; -using Microsoft.WindowsAzure.Storage.RetryPolicies; +using Azure; +using Azure.Core; +using Azure.Identity; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Specialized; namespace NuGetGallery { public class CloudBlobClientWrapper : ICloudBlobClient { + private const string SecondaryHostPostfix = "-secondary"; private readonly string _storageConnectionString; - private readonly BlobRequestOptions _defaultRequestOptions; - private readonly bool _readAccessGeoRedundant; - private CloudBlobClient _blobClient; + private readonly bool _readAccessGeoRedundant = false; + private readonly TimeSpan? _requestTimeout = null; + private readonly Lazy _primaryServiceUri; + private readonly Lazy _secondaryServiceUri; + private readonly Lazy _blobClient; + private readonly TokenCredential _tokenCredential = null; - public CloudBlobClientWrapper(string storageConnectionString, bool readAccessGeoRedundant) + public CloudBlobClientWrapper(string storageConnectionString, bool readAccessGeoRedundant = false, TimeSpan? requestTimeout = null) + : this(storageConnectionString) { - _storageConnectionString = storageConnectionString; _readAccessGeoRedundant = readAccessGeoRedundant; + _requestTimeout = requestTimeout; // OK to be null } - public CloudBlobClientWrapper(string storageConnectionString, BlobRequestOptions defaultRequestOptions) + private CloudBlobClientWrapper(string storageConnectionString, TokenCredential tokenCredential) + : this(storageConnectionString) { - _storageConnectionString = storageConnectionString; - _defaultRequestOptions = defaultRequestOptions; + _tokenCredential = tokenCredential ?? throw new ArgumentNullException(nameof(tokenCredential)); + } + + private CloudBlobClientWrapper(string storageConnectionString) + { + // workaround for https://github.com/Azure/azure-sdk-for-net/issues/44373 + _storageConnectionString = storageConnectionString.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + _primaryServiceUri = new Lazy(GetPrimaryServiceUri); + _secondaryServiceUri = new Lazy(GetSecondaryServiceUri); + _blobClient = new Lazy(CreateBlobServiceClient); + } + + public static CloudBlobClientWrapper UsingMsi(string storageConnectionString, string clientId = null) + { + var tokenCredential = new ManagedIdentityCredential(clientId); + return new CloudBlobClientWrapper(storageConnectionString, tokenCredential); } public ISimpleCloudBlob GetBlobFromUri(Uri uri) @@ -37,13 +58,13 @@ public ISimpleCloudBlob GetBlobFromUri(Uri uri) var uriBuilder = new UriBuilder(uri); uriBuilder.Query = string.Empty; - blob = new CloudBlobWrapper(new CloudBlockBlob( + blob = new CloudBlobWrapper(new BlockBlobClient( uriBuilder.Uri, - new StorageCredentials(uri.Query))); + new AzureSasCredential(uri.Query)), uri); } else { - blob = new CloudBlobWrapper(new CloudBlockBlob(uri)); + blob = new CloudBlobWrapper(new BlockBlobClient(uri), container: null); } return blob; @@ -51,21 +72,142 @@ public ISimpleCloudBlob GetBlobFromUri(Uri uri) public ICloudBlobContainer GetContainerReference(string containerAddress) { - if (_blobClient == null) + return new CloudBlobContainerWrapper(_blobClient.Value.GetBlobContainerClient(containerAddress), this); + } + + internal BlockBlobClient CreateBlockBlobClient(CloudBlobWrapper original, BlobClientOptions newOptions) + { + if (_readAccessGeoRedundant) + { + newOptions.GeoRedundantSecondaryUri = _secondaryServiceUri.Value; + } + if (_tokenCredential != null) { - _blobClient = CloudStorageAccount.Parse(_storageConnectionString).CreateCloudBlobClient(); + return new BlockBlobClient(original.Uri, _tokenCredential, newOptions); + } + return new BlockBlobClient(_storageConnectionString, original.Container, original.Name, newOptions); + } - if (_readAccessGeoRedundant) + internal BlobContainerClient CreateBlobContainerClient(string containerName, TimeSpan requestTimeout) + { + if (containerName == null) + { + throw new ArgumentNullException(nameof(containerName)); + } + + var newService = CreateBlobServiceClient(CreateBlobOptions(_readAccessGeoRedundant, requestTimeout)); + return newService.GetBlobContainerClient(containerName); + } + + internal BlobContainerClient CreateBlobContainerClient(CloudBlobLocationMode locationMode, string containerName, TimeSpan? requestTimeout = null) + { + if (containerName == null) + { + throw new ArgumentNullException(nameof(containerName)); + } + + if ((locationMode == CloudBlobLocationMode.PrimaryThenSecondary) + || (!_readAccessGeoRedundant && locationMode == CloudBlobLocationMode.PrimaryOnly)) + { + // Requested location mode is the same as we expect. + // If we are not supposed to be using RA-GRS, then there is no difference between PrimaryOnly and PrimaryThenSecondary + if (requestTimeout.HasValue) { - _blobClient.DefaultRequestOptions.LocationMode = LocationMode.PrimaryThenSecondary; + return CreateBlobContainerClient(containerName, requestTimeout.Value); } - else if (_defaultRequestOptions != null) + return null; + } + + if (locationMode == CloudBlobLocationMode.SecondaryOnly) + { + if (!_readAccessGeoRedundant) { - _blobClient.DefaultRequestOptions = _defaultRequestOptions; + throw new InvalidOperationException("Can't get secondary region for non RA-GRS storage services"); } + var service = CreateSecondaryBlobServiceClient(CreateBlobOptions(readAccessGeoRedundant: false, requestTimeout)); + return service.GetBlobContainerClient(containerName); } + if (locationMode == CloudBlobLocationMode.PrimaryOnly) + { + var service = CreateBlobServiceClient(CreateBlobOptions(readAccessGeoRedundant: false, requestTimeout)); + return service.GetBlobContainerClient(containerName); + } + throw new ArgumentOutOfRangeException(nameof(locationMode)); + } + + internal BlobServiceClient Client => _blobClient.Value; + internal bool UsingTokenCredential => _tokenCredential != null; - return new CloudBlobContainerWrapper(_blobClient.GetContainerReference(containerAddress)); + private Uri GetPrimaryServiceUri() + { + var tempClient = new BlobServiceClient(_storageConnectionString); + // if _storageConnectionString has SAS token, Uri will contain SAS signature, we need to strip it + var uriBuilder = new UriBuilder(tempClient.Uri); + uriBuilder.Query = ""; + uriBuilder.Fragment = ""; + return uriBuilder.Uri; + } + + private Uri GetSecondaryServiceUri() + { + var uriBuilder = new UriBuilder(_primaryServiceUri.Value); + var hostParts = uriBuilder.Host.Split('.'); + hostParts[0] = hostParts[0] + SecondaryHostPostfix; + uriBuilder.Host = string.Join(".", hostParts); + return uriBuilder.Uri; + } + + private BlobServiceClient CreateBlobServiceClient() + { + return CreateBlobServiceClient(CreateBlobOptions(_readAccessGeoRedundant)); + } + + private BlobClientOptions CreateBlobOptions(bool readAccessGeoRedundant, TimeSpan? requestTimeout = null) + { + var options = new BlobClientOptions(); + if (readAccessGeoRedundant) + { + options.GeoRedundantSecondaryUri = _secondaryServiceUri.Value; + } + if (requestTimeout.HasValue) + { + options.Retry.NetworkTimeout = requestTimeout.Value; + } + else if (_requestTimeout.HasValue) + { + options.Retry.NetworkTimeout = _requestTimeout.Value; + } + + return options; + } + + private BlobServiceClient CreateBlobServiceClient(BlobClientOptions options) + { + if (_tokenCredential != null) + { + return new BlobServiceClient(_primaryServiceUri.Value, _tokenCredential, options); + } + return new BlobServiceClient(_storageConnectionString, options); + } + + private BlobServiceClient CreateSecondaryBlobServiceClient(BlobClientOptions options) + { + if (_tokenCredential != null) + { + return new BlobServiceClient(_secondaryServiceUri.Value, _tokenCredential, options); + } + string secondaryConnectionString = GetSecondaryConnectionString(); + return new BlobServiceClient(secondaryConnectionString, options); + } + + private string GetSecondaryConnectionString() + { + var primaryAccountName = _primaryServiceUri.Value.Host.Split('.')[0]; + var secondaryAccountName = _secondaryServiceUri.Value.Host.Split('.')[0]; + var secondaryConnectionString = _storageConnectionString + .Replace($"https://{primaryAccountName}.", $"https://{secondaryAccountName}.") + .Replace($"AccountName={primaryAccountName};", $"AccountName={secondaryAccountName};"); + return secondaryConnectionString; } } } \ No newline at end of file diff --git a/src/NuGetGallery.Core/Services/CloudBlobConflictException.cs b/src/NuGetGallery.Core/Services/CloudBlobConflictException.cs new file mode 100644 index 0000000000..99cdc95575 --- /dev/null +++ b/src/NuGetGallery.Core/Services/CloudBlobConflictException.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGetGallery +{ + public class CloudBlobConflictException : CloudBlobStorageException + { + public CloudBlobConflictException(Exception innerException) + : base(innerException) + { + } + } +} diff --git a/src/NuGetGallery.Core/Services/CloudBlobContainerNotFoundException.cs b/src/NuGetGallery.Core/Services/CloudBlobContainerNotFoundException.cs new file mode 100644 index 0000000000..fcd4b7c3cb --- /dev/null +++ b/src/NuGetGallery.Core/Services/CloudBlobContainerNotFoundException.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGetGallery +{ + public class CloudBlobContainerNotFoundException : CloudBlobGenericNotFoundException + { + public CloudBlobContainerNotFoundException(Exception innerException) + : base("Container not found", innerException) + { + } + } +} diff --git a/src/NuGetGallery.Core/Services/CloudBlobContainerWrapper.cs b/src/NuGetGallery.Core/Services/CloudBlobContainerWrapper.cs index d14f809bc4..75d05a60b3 100644 --- a/src/NuGetGallery.Core/Services/CloudBlobContainerWrapper.cs +++ b/src/NuGetGallery.Core/Services/CloudBlobContainerWrapper.cs @@ -1,89 +1,117 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized; namespace NuGetGallery { public class CloudBlobContainerWrapper : ICloudBlobContainer { - private readonly CloudBlobContainer _blobContainer; + private readonly CloudBlobClientWrapper _account; + private readonly BlobContainerClient _blobContainer; - public CloudBlobContainerWrapper(CloudBlobContainer blobContainer) + public CloudBlobContainerWrapper(BlobContainerClient blobContainer, CloudBlobClientWrapper account) { - _blobContainer = blobContainer; + _blobContainer = blobContainer ?? throw new ArgumentNullException(nameof(blobContainer)); + _account = account ?? throw new ArgumentNullException(nameof(account)); } public async Task ListBlobsSegmentedAsync( string prefix, bool useFlatBlobListing, - BlobListingDetails blobListingDetails, + ListingDetails blobListingDetails, int? maxResults, - BlobContinuationToken blobContinuationToken, - BlobRequestOptions options, - OperationContext operationContext, + BlobListContinuationToken blobContinuationToken, + TimeSpan? requestTimeout, + CloudBlobLocationMode? cloudBlobLocationMode, CancellationToken cancellationToken) { - var segment = await _blobContainer.ListBlobsSegmentedAsync( - prefix, - useFlatBlobListing, - blobListingDetails, - maxResults, - blobContinuationToken, - options, - operationContext, - cancellationToken); - - return new BlobResultSegmentWrapper(segment); - } - - public Task CreateIfNotExistAsync(BlobContainerPermissions permissions) - { - var publicAccess = permissions?.PublicAccess; + string continuationToken = blobContinuationToken?.ContinuationToken; - if (publicAccess.HasValue) + BlobContainerClient blobContainerClient = _blobContainer; + if (cloudBlobLocationMode.HasValue) { - return _blobContainer.CreateIfNotExistsAsync(publicAccess.Value, options: null, operationContext: null); + blobContainerClient = _account.CreateBlobContainerClient(cloudBlobLocationMode.Value, _blobContainer.Name, requestTimeout) ?? blobContainerClient; + } + else if (requestTimeout.HasValue) + { + blobContainerClient = _account.CreateBlobContainerClient(_blobContainer.Name, requestTimeout.Value); } - return _blobContainer.CreateIfNotExistsAsync(); - } + BlobTraits traits = CloudWrapperHelpers.GetSdkBlobTraits(blobListingDetails); + BlobStates states = CloudWrapperHelpers.GetSdkBlobStates(blobListingDetails); + var enumerable = blobContainerClient + .GetBlobsAsync(traits: traits, states: states, prefix: prefix, cancellationToken: cancellationToken) + .AsPages(continuationToken, maxResults); - public async Task SetPermissionsAsync(BlobContainerPermissions permissions) + var enumerator = enumerable.GetAsyncEnumerator(cancellationToken); + try + { + if (await CloudWrapperHelpers.WrapStorageExceptionAsync(() => enumerator.MoveNextAsync().AsTask())) + { + var page = enumerator.Current; + var nextContinuationToken = string.IsNullOrEmpty(page.ContinuationToken) ? null : page.ContinuationToken; + return new BlobResultSegmentWrapper(page.Values.Select(x => GetBlobReference(x)).ToList(), nextContinuationToken); + } + } + finally + { + await enumerator.DisposeAsync(); + } + + return new BlobResultSegmentWrapper(new List(), null); + } + + public async Task CreateIfNotExistAsync(bool enablePublicAccess) { - await _blobContainer.SetPermissionsAsync(permissions); + var accessType = enablePublicAccess ? PublicAccessType.Blob : PublicAccessType.None; + + await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blobContainer.CreateIfNotExistsAsync(accessType)); } public ISimpleCloudBlob GetBlobReference(string blobAddressUri) { - return new CloudBlobWrapper(_blobContainer.GetBlockBlobReference(blobAddressUri)); + return new CloudBlobWrapper(_blobContainer.GetBlockBlobClient(blobAddressUri), this); + } + + private ISimpleCloudBlob GetBlobReference(BlobItem item) + { + return new CloudBlobWrapper(_blobContainer.GetBlockBlobClient(item.Name), item, this); } - public async Task ExistsAsync(BlobRequestOptions blobRequestOptions, OperationContext context) + public async Task ExistsAsync(CloudBlobLocationMode? cloudBlobLocationMode) { - return await _blobContainer.ExistsAsync(blobRequestOptions, context); + BlobContainerClient containerClient = _blobContainer; + if (cloudBlobLocationMode.HasValue) + { + containerClient = _account.CreateBlobContainerClient(cloudBlobLocationMode.Value, _blobContainer.Name) ?? containerClient; + } + return (await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + containerClient.ExistsAsync())).Value; } public async Task DeleteIfExistsAsync() { - return await _blobContainer.DeleteIfExistsAsync(); + return await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blobContainer.DeleteIfExistsAsync()); } - public async Task CreateAsync(BlobContainerPermissions permissions) + public async Task CreateAsync(bool enablePublicAccess) { - var publicAccess = permissions?.PublicAccess; + var accessType = enablePublicAccess ? PublicAccessType.Blob : PublicAccessType.None; - if (publicAccess.HasValue) - { - await _blobContainer.CreateAsync(publicAccess.Value, options: null, operationContext: null); - } - else - { - await _blobContainer.CreateAsync(); - } + await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blobContainer.CreateAsync(accessType)); } + + internal CloudBlobClientWrapper Account => _account; } } diff --git a/src/NuGetGallery.Core/Services/CloudBlobCopyState.cs b/src/NuGetGallery.Core/Services/CloudBlobCopyState.cs new file mode 100644 index 0000000000..20945a20bd --- /dev/null +++ b/src/NuGetGallery.Core/Services/CloudBlobCopyState.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGetGallery +{ + internal class CloudBlobCopyState : ICloudBlobCopyState + { + private readonly CloudBlobWrapper _blob; + + public CloudBlobCopyState(CloudBlobWrapper blob) + { + _blob = blob ?? throw new ArgumentNullException(nameof(blob)); + } + public CloudBlobCopyStatus Status => CloudWrapperHelpers.GetBlobCopyStatus(_blob.BlobProperties?.CopyStatus); + + public string StatusDescription => _blob.BlobProperties?.CopyStatusDescription; + } +} diff --git a/src/NuGetGallery.Core/Services/CloudBlobCopyStatus.cs b/src/NuGetGallery.Core/Services/CloudBlobCopyStatus.cs new file mode 100644 index 0000000000..1360d9ece0 --- /dev/null +++ b/src/NuGetGallery.Core/Services/CloudBlobCopyStatus.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGetGallery +{ + public enum CloudBlobCopyStatus + { + None, + Pending, + Success, + Aborted, + Failed + } +} diff --git a/src/NuGetGallery.Core/Services/CloudBlobCoreFileStorageService.cs b/src/NuGetGallery.Core/Services/CloudBlobCoreFileStorageService.cs index 272e80cfb1..3c4f072df7 100644 --- a/src/NuGetGallery.Core/Services/CloudBlobCoreFileStorageService.cs +++ b/src/NuGetGallery.Core/Services/CloudBlobCoreFileStorageService.cs @@ -9,9 +9,6 @@ using System.IO; using System.Net; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; -using Microsoft.WindowsAzure.Storage.Blob.Protocol; using NuGetGallery.Diagnostics; using LogLevel = Microsoft.Extensions.Logging.LogLevel; @@ -158,11 +155,6 @@ private async Task CopyFileAsync( var destContainer = await GetContainerAsync(destFolderName); var destBlob = destContainer.GetBlobReference(destFileName); destAccessCondition = destAccessCondition ?? AccessConditionWrapper.GenerateIfNotExistsCondition(); - var mappedDestAccessCondition = new AccessCondition - { - IfNoneMatchETag = destAccessCondition.IfNoneMatchETag, - IfMatchETag = destAccessCondition.IfMatchETag, - }; if (!await srcBlob.ExistsAsync()) { @@ -174,14 +166,15 @@ private async Task CopyFileAsync( // Determine the source blob etag. await srcBlob.FetchAttributesAsync(); - var srcAccessCondition = AccessCondition.GenerateIfMatchCondition(srcBlob.ETag); + var srcAccessCondition = AccessConditionWrapper.GenerateIfMatchCondition(srcBlob.ETag); // Check if the destination blob already exists and fetch attributes. if (await destBlob.ExistsAsync()) { var sourceBlobMetadata = srcBlob.Metadata; + await destBlob.FetchAttributesAsync(); var destinationBlobMetadata = destBlob.Metadata; - if (destBlob.CopyState?.Status == CopyStatus.Failed) + if (destBlob.CopyState?.Status == CloudBlobCopyStatus.Failed) { // If the last copy failed, allow this copy to occur no matter what the caller's destination // condition is. This is because the source blob is preferable over a failed copy. We use the etag @@ -193,7 +186,7 @@ private async Task CopyFileAsync( message: $"Destination blob '{destFolderName}/{destFileName}' already exists but has a " + $"failed copy status. This blob will be replaced if the etag matches '{destBlob.ETag}'."); - mappedDestAccessCondition = AccessCondition.GenerateIfMatchCondition(destBlob.ETag); + destAccessCondition = AccessConditionWrapper.GenerateIfMatchCondition(destBlob.ETag); } else if (sourceBlobMetadata != null && destinationBlobMetadata != null) { @@ -233,7 +226,7 @@ private async Task CopyFileAsync( eventId: 0, message: $"Copying of source blob '{srcBlob.Uri}' to '{destFolderName}/{destFileName}' with source " + $"access condition {Log(srcAccessCondition)} and destination access condition " + - $"{Log(mappedDestAccessCondition)}."); + $"{Log(destAccessCondition)}."); // Start the server-side copy and wait for it to complete. If "If-None-Match: *" was specified and the // destination already exists, HTTP 409 is thrown. If "If-Match: ETAG" was specified and the destination @@ -243,9 +236,9 @@ private async Task CopyFileAsync( await destBlob.StartCopyAsync( srcBlob, srcAccessCondition, - mappedDestAccessCondition); + destAccessCondition); } - catch (StorageException ex) when (ex.IsFileAlreadyExistsException()) + catch (CloudBlobConflictException ex) { throw new FileAlreadyExistsException( string.Format( @@ -253,11 +246,11 @@ await destBlob.StartCopyAsync( "There is already a blob with name {0} in container {1}.", destFileName, destFolderName), - ex); + ex.InnerException); } var stopwatch = Stopwatch.StartNew(); - while (destBlob.CopyState.Status == CopyStatus.Pending + while (destBlob.CopyState.Status == CloudBlobCopyStatus.Pending && stopwatch.Elapsed < MaxCopyDuration) { if (!await destBlob.ExistsAsync()) @@ -272,19 +265,19 @@ await destBlob.StartCopyAsync( await Task.Delay(CopyPollFrequency); } - if (destBlob.CopyState.Status == CopyStatus.Pending) + if (destBlob.CopyState.Status == CloudBlobCopyStatus.Pending) { throw new TimeoutException($"Waiting for the blob copy operation to complete timed out after {MaxCopyDuration.TotalSeconds} seconds."); } - else if (destBlob.CopyState.Status != CopyStatus.Success) + else if (destBlob.CopyState.Status != CloudBlobCopyStatus.Success) { - throw new StorageException($"The blob copy operation had copy status {destBlob.CopyState.Status} ({destBlob.CopyState.StatusDescription})."); + throw new CloudBlobStorageException($"The blob copy operation had copy status {destBlob.CopyState.Status} ({destBlob.CopyState.StatusDescription})."); } return srcBlob.ETag; } - private static string Log(AccessCondition accessCondition) + private static string Log(IAccessCondition accessCondition) { if (accessCondition?.IfMatchETag != null) { @@ -318,7 +311,7 @@ public async Task SaveFileAsync(string folderName, string fileName, string conte { await blob.UploadFromStreamAsync(file, overwrite); } - catch (StorageException ex) when (ex.IsFileAlreadyExistsException()) + catch (CloudBlobConflictException ex) { throw new FileAlreadyExistsException( string.Format( @@ -326,7 +319,7 @@ public async Task SaveFileAsync(string folderName, string fileName, string conte "There is already a blob with name {0} in container {1}.", fileName, folderName), - ex); + ex.InnerException); } blob.Properties.ContentType = contentType; @@ -341,17 +334,11 @@ public async Task SaveFileAsync(string folderName, string fileName, Stream file, accessConditions = accessConditions ?? AccessConditionWrapper.GenerateIfNotExistsCondition(); - var mappedAccessCondition = new AccessCondition - { - IfNoneMatchETag = accessConditions.IfNoneMatchETag, - IfMatchETag = accessConditions.IfMatchETag, - }; - try { - await blob.UploadFromStreamAsync(file, mappedAccessCondition); + await blob.UploadFromStreamAsync(file, accessConditions); } - catch (StorageException ex) when (ex.IsFileAlreadyExistsException()) + catch (CloudBlobConflictException ex) { throw new FileAlreadyExistsException( string.Format( @@ -359,7 +346,7 @@ public async Task SaveFileAsync(string folderName, string fileName, Stream file, "There is already a blob with name {0} in container {1}.", fileName, folderName), - ex); + ex.InnerException); } blob.Properties.ContentType = GetContentType(folderName); @@ -373,7 +360,7 @@ public async Task GetFileUriAsync(string folderName, string fileName) return blob.Uri; } - public async Task GetPriviledgedFileUriAsync( + public async Task GetPrivilegedFileUriAsync( string folderName, string fileName, FileUriPermissions permissions, @@ -385,10 +372,9 @@ public async Task GetPriviledgedFileUriAsync( } var blob = await GetBlobForUriAsync(folderName, fileName); + string sas = await blob.GetSharedAccessSignature(permissions, endOfAccess); - return new Uri( - blob.Uri, - blob.GetSharedAccessSignature(MapFileUriPermissions(permissions), endOfAccess)); + return new Uri(blob.Uri, sas); } public async Task GetFileReadUriAsync(string folderName, string fileName, DateTimeOffset? endOfAccess) @@ -410,9 +396,9 @@ public async Task GetFileReadUriAsync(string folderName, string fileName, D throw new ArgumentOutOfRangeException(nameof(endOfAccess), $"{nameof(endOfAccess)} is in the past"); } - return new Uri( - blob.Uri, - blob.GetSharedAccessSignature(SharedAccessBlobPermissions.Read, endOfAccess)); + string sas = await blob.GetSharedAccessSignature(FileUriPermissions.Read, endOfAccess.Value); + + return new Uri(blob.Uri, sas); } /// @@ -454,13 +440,7 @@ public async Task SetMetadataAsync( if (wasUpdated) { var accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blob.ETag); - var mappedAccessCondition = new AccessCondition - { - IfNoneMatchETag = accessCondition.IfNoneMatchETag, - IfMatchETag = accessCondition.IfMatchETag - }; - - await blob.SetMetadataAsync(mappedAccessCondition); + await blob.SetMetadataAsync(accessCondition); } } @@ -475,7 +455,7 @@ public async Task SetMetadataAsync( public async Task SetPropertiesAsync( string folderName, string fileName, - Func>, BlobProperties, Task> updatePropertiesAsync) + Func>, ICloudBlobProperties, Task> updatePropertiesAsync) { if (folderName == null) { @@ -503,13 +483,7 @@ public async Task SetPropertiesAsync( if (wasUpdated) { var accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blob.ETag); - var mappedAccessCondition = new AccessCondition - { - IfNoneMatchETag = accessCondition.IfNoneMatchETag, - IfMatchETag = accessCondition.IfMatchETag - }; - - await blob.SetPropertiesAsync(mappedAccessCondition); + await blob.SetPropertiesAsync(accessCondition); } } @@ -528,17 +502,12 @@ public async Task GetETagOrNullAsync( return blob.ETag; } // In case that the blob does not exist return null. - catch (StorageException) + catch (CloudBlobStorageException) { return null; } } - private static SharedAccessBlobPermissions MapFileUriPermissions(FileUriPermissions permissions) - { - return (SharedAccessBlobPermissions)permissions; - } - private async Task GetBlobForUriAsync(string folderName, string fileName) { folderName = folderName ?? throw new ArgumentNullException(nameof(folderName)); @@ -581,35 +550,17 @@ await blob.DownloadToStreamAsync( accessCondition: ifNoneMatch == null ? null : - AccessCondition.GenerateIfNoneMatchCondition(ifNoneMatch)); + AccessConditionWrapper.GenerateIfNoneMatchCondition(ifNoneMatch)); } - catch (StorageException ex) + catch (CloudBlobNotModifiedException) { stream.Dispose(); - - if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotModified) - { - return new StorageResult(HttpStatusCode.NotModified, null, blob.ETag); - } - else if (ex.RequestInformation.ExtendedErrorInformation?.ErrorCode == BlobErrorCodeStrings.BlobNotFound) - { - return new StorageResult(HttpStatusCode.NotFound, null, blob.ETag); - } - - throw; + return new StorageResult(HttpStatusCode.NotModified, null, blob.ETag); } - catch (TestableStorageClientException ex) + catch (CloudBlobNotFoundException) { - // This is for unit test only, because we can't construct an - // StorageException object with the required ErrorCode stream.Dispose(); - - if (ex.ErrorCode == BlobErrorCodeStrings.BlobNotFound) - { - return new StorageResult(HttpStatusCode.NotFound, null, blob.ETag); - } - - throw; + return new StorageResult(HttpStatusCode.NotFound, null, blob.ETag); } stream.Position = 0; @@ -629,12 +580,7 @@ private string GetCacheControl(string folderName) private async Task PrepareContainer(string folderName, bool isPublic) { var container = _client.GetContainerReference(folderName); - var permissions = new BlobContainerPermissions - { - PublicAccess = isPublic ? BlobContainerPublicAccessType.Blob : BlobContainerPublicAccessType.Off - }; - - await container.CreateIfNotExistAsync(permissions); + await container.CreateIfNotExistAsync(isPublic); return container; } diff --git a/src/NuGetGallery.Core/Services/CloudBlobGenericNotFoundException.cs b/src/NuGetGallery.Core/Services/CloudBlobGenericNotFoundException.cs new file mode 100644 index 0000000000..e06d77713e --- /dev/null +++ b/src/NuGetGallery.Core/Services/CloudBlobGenericNotFoundException.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGetGallery +{ + public class CloudBlobGenericNotFoundException : CloudBlobStorageException + { + public CloudBlobGenericNotFoundException(Exception innerException) + : base(innerException) + { + } + + public CloudBlobGenericNotFoundException(string message, Exception innerException) + : base(message, innerException) + { + } + } +} diff --git a/src/NuGetGallery.Core/Services/CloudBlobLocationMode.cs b/src/NuGetGallery.Core/Services/CloudBlobLocationMode.cs new file mode 100644 index 0000000000..5d9aa4b81c --- /dev/null +++ b/src/NuGetGallery.Core/Services/CloudBlobLocationMode.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGetGallery +{ + public enum CloudBlobLocationMode + { + PrimaryOnly, + PrimaryThenSecondary, + SecondaryOnly, + } +} diff --git a/src/NuGetGallery.Core/Services/CloudBlobNotFoundException.cs b/src/NuGetGallery.Core/Services/CloudBlobNotFoundException.cs new file mode 100644 index 0000000000..e7ffe1674e --- /dev/null +++ b/src/NuGetGallery.Core/Services/CloudBlobNotFoundException.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGetGallery +{ + public class CloudBlobNotFoundException : CloudBlobGenericNotFoundException + { + public CloudBlobNotFoundException(Exception innerException) + : base("Blob not found", innerException) + { + } + } +} diff --git a/src/NuGetGallery.Core/Services/CloudBlobNotModifiedException.cs b/src/NuGetGallery.Core/Services/CloudBlobNotModifiedException.cs new file mode 100644 index 0000000000..6caadf7eb3 --- /dev/null +++ b/src/NuGetGallery.Core/Services/CloudBlobNotModifiedException.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGetGallery +{ + public class CloudBlobNotModifiedException : CloudBlobStorageException + { + public CloudBlobNotModifiedException(Exception innerException) + : base(innerException) + { + } + } +} diff --git a/src/NuGetGallery.Core/Services/CloudBlobPreconditionFailedException.cs b/src/NuGetGallery.Core/Services/CloudBlobPreconditionFailedException.cs new file mode 100644 index 0000000000..d3dcae0850 --- /dev/null +++ b/src/NuGetGallery.Core/Services/CloudBlobPreconditionFailedException.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGetGallery +{ + public class CloudBlobPreconditionFailedException : CloudBlobStorageException + { + public CloudBlobPreconditionFailedException(Exception innerException) + : base(innerException) + { + } + } +} diff --git a/src/NuGetGallery.Core/Services/CloudBlobPropertiesWrapper.cs b/src/NuGetGallery.Core/Services/CloudBlobPropertiesWrapper.cs new file mode 100644 index 0000000000..3941462dd4 --- /dev/null +++ b/src/NuGetGallery.Core/Services/CloudBlobPropertiesWrapper.cs @@ -0,0 +1,66 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Azure.Storage.Blobs.Models; + +namespace NuGetGallery +{ + internal class CloudBlobPropertiesWrapper : ICloudBlobProperties + { + private readonly CloudBlobWrapper _blob; + + public CloudBlobPropertiesWrapper(CloudBlobWrapper cloudBlobWrapper) + { + _blob = cloudBlobWrapper ?? throw new ArgumentNullException(nameof(cloudBlobWrapper)); + } + + public DateTimeOffset? LastModified => _blob.BlobProperties.LastModifiedUtc; + + public long Length => _blob.BlobProperties.Length; + + public string ContentType + { + get => _blob.BlobHeaders.ContentType; + set => SafeHeaders.ContentType = value; + } + + public string ContentEncoding + { + get => _blob.BlobHeaders.ContentEncoding; + set => SafeHeaders.ContentEncoding = value; + } + + public string CacheControl + { + get => _blob.BlobHeaders.CacheControl; + set => SafeHeaders.CacheControl = value; + } + + public string ContentMD5 + { + get => ToBase64String(_blob.BlobHeaders.ContentHash); + } + + private BlobHttpHeaders SafeHeaders + { + get + { + if (_blob.BlobHeaders == null) + { + _blob.BlobHeaders = new BlobHttpHeaders(); + } + return _blob.BlobHeaders; + } + } + + private static string ToBase64String(byte[] array) + { + if (array == null) + { + return null; + } + return Convert.ToBase64String(array); + } + } +} diff --git a/src/NuGetGallery.Core/Services/CloudBlobReadOnlyProperties.cs b/src/NuGetGallery.Core/Services/CloudBlobReadOnlyProperties.cs new file mode 100644 index 0000000000..1ec5e5189e --- /dev/null +++ b/src/NuGetGallery.Core/Services/CloudBlobReadOnlyProperties.cs @@ -0,0 +1,35 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Azure.Storage.Blobs.Models; + +namespace NuGetGallery +{ + internal class CloudBlobReadOnlyProperties + { + public DateTime LastModifiedUtc { get; } + public long Length { get; } + public bool IsSnapshot { get; } + public CopyStatus? CopyStatus { get; } + public string CopyStatusDescription { get; } + + public CloudBlobReadOnlyProperties(BlobProperties blobProperties, bool isSnapshot = false) + { + LastModifiedUtc = blobProperties.LastModified.UtcDateTime; + Length = blobProperties.ContentLength; + IsSnapshot = isSnapshot; + CopyStatus = blobProperties.BlobCopyStatus; + CopyStatusDescription = blobProperties.CopyStatusDescription; + } + + public CloudBlobReadOnlyProperties(BlobItem blobItem) + { + LastModifiedUtc = blobItem.Properties?.LastModified?.UtcDateTime ?? DateTime.MinValue; + Length = blobItem.Properties?.ContentLength ?? 0; + IsSnapshot = blobItem.Snapshot != null; + CopyStatus = null; + CopyStatusDescription = null; + } + } +} diff --git a/src/NuGetGallery.Core/Services/CloudBlobStorageException.cs b/src/NuGetGallery.Core/Services/CloudBlobStorageException.cs new file mode 100644 index 0000000000..168a5ac2d1 --- /dev/null +++ b/src/NuGetGallery.Core/Services/CloudBlobStorageException.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGetGallery +{ + public class CloudBlobStorageException : Exception + { + public CloudBlobStorageException(Exception innerException) + : base(innerException?.Message ?? string.Empty, innerException) + { + } + + public CloudBlobStorageException(string message, Exception innerException) + : base(message, innerException) + { + } + + public CloudBlobStorageException(string message) + : base(message) + { + } + } +} diff --git a/src/NuGetGallery.Core/Services/CloudBlobWrapper.cs b/src/NuGetGallery.Core/Services/CloudBlobWrapper.cs index e9bf206dbc..725566a067 100644 --- a/src/NuGetGallery.Core/Services/CloudBlobWrapper.cs +++ b/src/NuGetGallery.Core/Services/CloudBlobWrapper.cs @@ -7,28 +7,81 @@ using System.Net; using System.Threading; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; -using Microsoft.WindowsAzure.Storage.RetryPolicies; +using Azure; +using Azure.Core; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized; +using Azure.Storage.Sas; namespace NuGetGallery { public class CloudBlobWrapper : ISimpleCloudBlob { - private readonly CloudBlockBlob _blob; - - public BlobProperties Properties => _blob.Properties; - public IDictionary Metadata => _blob.Metadata; - public CopyState CopyState => _blob.CopyState; - public Uri Uri => _blob.Uri; + private const string ContentDispositionHeaderName = "Content-Disposition"; + private const string ContentEncodingHeaderName = "Content-Encoding"; + private const string ContentLanguageHeaderName = "Content-Language"; + private const string CacheControlHeaderName = "Cache-Control"; + private const string ContentMd5HeaderName = "Content-Md5"; + private readonly BlockBlobClient _blob; + private readonly CloudBlobContainerWrapper _container; + private string _lastSeenEtag = null; + + public ICloudBlobProperties Properties { get; private set; } + public IDictionary Metadata { get; private set; } + public ICloudBlobCopyState CopyState { get; private set; } + public Uri Uri + { + get + { + var builder = new UriBuilder(_blob.Uri); + builder.Query = string.Empty; + return builder.Uri; + } + } public string Name => _blob.Name; - public DateTime LastModifiedUtc => _blob.Properties.LastModified?.UtcDateTime ?? DateTime.MinValue; - public string ETag => _blob.Properties.ETag; - public bool IsSnapshot => _blob.IsSnapshot; + public string Container => _blob.BlobContainerName; + public DateTime LastModifiedUtc => BlobProperties.LastModifiedUtc; + public string ETag => _lastSeenEtag; + public bool IsSnapshot => BlobProperties.IsSnapshot; + + internal Uri BlobSasUri { get; } = null; + internal CloudBlobReadOnlyProperties BlobProperties { get; set; } = null; + internal BlobHttpHeaders BlobHeaders { get; set; } = null; - public CloudBlobWrapper(CloudBlockBlob blob) + public CloudBlobWrapper(BlockBlobClient blob, CloudBlobContainerWrapper container) { - _blob = blob; + _blob = blob ?? throw new ArgumentNullException(nameof(blob)); + _container = container; // container can be null + + Properties = new CloudBlobPropertiesWrapper(this); + CopyState = new CloudBlobCopyState(this); + } + + public CloudBlobWrapper(BlockBlobClient blob, BlobItem blobData, CloudBlobContainerWrapper container) + : this(blob, container) + { + if (blobData != null) + { + ReplaceMetadata(blobData.Metadata); + BlobProperties = new CloudBlobReadOnlyProperties(blobData); + if (blobData.Properties != null) + { + BlobHeaders = new BlobHttpHeaders(); + BlobHeaders.ContentType = blobData.Properties.ContentType; + BlobHeaders.ContentDisposition = blobData.Properties.ContentDisposition; + BlobHeaders.ContentEncoding = blobData.Properties.ContentEncoding; + BlobHeaders.ContentLanguage = blobData.Properties.ContentLanguage; + BlobHeaders.CacheControl = blobData.Properties.CacheControl; + BlobHeaders.ContentHash = blobData.Properties.ContentHash; + } + } + } + + internal CloudBlobWrapper(BlockBlobClient blob, Uri blobSasUri) + : this(blob, container: null) + { + BlobSasUri = blobSasUri ?? throw new ArgumentNullException(nameof(blobSasUri)); } public static CloudBlobWrapper FromUri(Uri uri) @@ -43,33 +96,56 @@ public static CloudBlobWrapper FromUri(Uri uri) throw new ArgumentException($"{nameof(uri)} must point to blob storage", nameof(uri)); } - var blob = new CloudBlockBlob(uri); - return new CloudBlobWrapper(blob); + var blob = new BlockBlobClient(uri); + return new CloudBlobWrapper(blob, container: null); } - public async Task OpenReadAsync(AccessCondition accessCondition) + public async Task OpenReadAsync(IAccessCondition accessCondition) { - return await _blob.OpenReadAsync( - accessCondition: accessCondition, - options: null, - operationContext: null); + BlobDownloadOptions options = null; + if (accessCondition != null) + { + options = new BlobDownloadOptions() + { + Conditions = CloudWrapperHelpers.GetSdkAccessCondition(accessCondition), + }; + } + var result = await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blob.DownloadStreamingAsync(options)); + if (result.GetRawResponse().Status == (int)HttpStatusCode.NotModified) + { + // calling code expects an exception thrown on not modified response + throw new CloudBlobNotModifiedException(null); + } + UpdateEtag(result.Value.Details); + return result.Value.Content; } - public async Task OpenWriteAsync(AccessCondition accessCondition) + public async Task OpenWriteAsync(IAccessCondition accessCondition, string contentType = null) { - return await _blob.OpenWriteAsync( - accessCondition: accessCondition, - options: null, - operationContext: null); + BlockBlobOpenWriteOptions options = new BlockBlobOpenWriteOptions + { + OpenConditions = CloudWrapperHelpers.GetSdkAccessCondition(accessCondition), + }; + if (contentType != null) + { + options.HttpHeaders = new BlobHttpHeaders + { + ContentType = contentType, + }; + } + return await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + // overwrite must be set to true for BlockBlobClient.OpenWriteAsync call *shrug* + // The value itself does not seem to be otherwise used anywhere. + // https://github.com/Azure/azure-sdk-for-net/blob/aec1a1389636a2ef76270ab4bdcb0715a2abb1aa/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs#L2776-L2779 + // https://github.com/Azure/azure-sdk-for-net/blob/aec1a1389636a2ef76270ab4bdcb0715a2abb1aa/sdk/storage/Azure.Storage.Blobs/tests/BlobClientOpenWriteTests.cs#L124-L133 + _blob.OpenWriteAsync(overwrite: true, options)); } public async Task DeleteIfExistsAsync() { - await _blob.DeleteIfExistsAsync( - DeleteSnapshotsOption.IncludeSnapshots, - accessCondition: null, - options: null, - operationContext: null); + await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots)); } public Task DownloadToStreamAsync(Stream target) @@ -77,89 +153,212 @@ public Task DownloadToStreamAsync(Stream target) return DownloadToStreamAsync(target, accessCondition: null); } - public async Task DownloadToStreamAsync(Stream target, AccessCondition accessCondition) + public async Task DownloadToStreamAsync(Stream target, IAccessCondition accessCondition) { - // Note: Overloads of FromAsync that take an AsyncCallback and State to pass through are more efficient: - // http://blogs.msdn.com/b/pfxteam/archive/2009/06/09/9716439.aspx - var options = new BlobRequestOptions() + // 304s are not retried with Azure.Storage.Blobs, so no need for custom retry policy. + + BlobDownloadToOptions downloadOptions = null; + if (accessCondition != null) { - // The default retry policy treats a 304 as an error that requires a retry. We don't want that! - RetryPolicy = new DontRetryOnNotModifiedPolicy(new LinearRetry()) - }; + downloadOptions = new BlobDownloadToOptions + { + Conditions = CloudWrapperHelpers.GetSdkAccessCondition(accessCondition), + }; + } - await _blob.DownloadToStreamAsync(target, accessCondition, options, operationContext: null); + var response = UpdateEtag(await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blob.DownloadToAsync(target, downloadOptions))); + + if (response.Status == (int)HttpStatusCode.NotModified) + { + // calling code expects an exception thrown on not modified response + throw new CloudBlobNotModifiedException(null); + } } public async Task ExistsAsync() { - return await _blob.ExistsAsync(); + return await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blob.ExistsAsync()); } public async Task SnapshotAsync(CancellationToken token) { - await _blob.SnapshotAsync( - metadata: null, - accessCondition: null, - options: null, - operationContext: null, - token); + await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blob.CreateSnapshotAsync(cancellationToken: token)); } public async Task SetPropertiesAsync() { - await _blob.SetPropertiesAsync(); + UpdateEtag(await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blob.SetHttpHeadersAsync(BlobHeaders))); } - public async Task SetPropertiesAsync(AccessCondition accessCondition) + public async Task SetPropertiesAsync(IAccessCondition accessCondition) { - await _blob.SetPropertiesAsync(accessCondition, options: null, operationContext: null); + UpdateEtag(await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blob.SetHttpHeadersAsync( + BlobHeaders, + CloudWrapperHelpers.GetSdkAccessCondition(accessCondition)))); } - public async Task SetMetadataAsync(AccessCondition accessCondition) + public async Task SetMetadataAsync(IAccessCondition accessCondition) { - await _blob.SetMetadataAsync(accessCondition, options: null, operationContext: null); + UpdateEtag(await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blob.SetMetadataAsync( + Metadata, + CloudWrapperHelpers.GetSdkAccessCondition(accessCondition)))); } public async Task UploadFromStreamAsync(Stream source, bool overwrite) { if (overwrite) { - await _blob.UploadFromStreamAsync(source); + BlobUploadOptions options = null; + if (BlobHeaders != null) + { + options = new BlobUploadOptions + { + HttpHeaders = BlobHeaders, + }; + } + UpdateEtag(await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blob.UploadAsync(source))); + await FetchAttributesAsync(); } else { - await UploadFromStreamAsync(source, AccessCondition.GenerateIfNoneMatchCondition("*")); + await UploadFromStreamAsync(source, AccessConditionWrapper.GenerateIfNoneMatchCondition("*")); } } - public async Task UploadFromStreamAsync(Stream source, AccessCondition accessCondition) + public async Task UploadFromStreamAsync(Stream source, IAccessCondition accessCondition) { - await _blob.UploadFromStreamAsync( - source, - accessCondition, - options: null, - operationContext: null); + BlobUploadOptions options = null; + if (accessCondition != null || BlobHeaders != null) + { + options = new BlobUploadOptions + { + Conditions = CloudWrapperHelpers.GetSdkAccessCondition(accessCondition), + HttpHeaders = BlobHeaders, + }; + } + UpdateEtag(await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blob.UploadAsync(source, options))); + await FetchAttributesAsync(); } public async Task FetchAttributesAsync() { - await _blob.FetchAttributesAsync(); + var blobProperties = UpdateEtag(await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blob.GetPropertiesAsync())).Value; + BlobProperties = new CloudBlobReadOnlyProperties(blobProperties); + ReplaceHttpHeaders(blobProperties); + ReplaceMetadata(blobProperties.Metadata); } - public string GetSharedAccessSignature(SharedAccessBlobPermissions permissions, DateTimeOffset? endOfAccess) + private void ReplaceHttpHeaders(BlobProperties blobProperties) { - var accessPolicy = new SharedAccessBlobPolicy + if (BlobHeaders == null) { - SharedAccessExpiryTime = endOfAccess, - Permissions = permissions, - }; + BlobHeaders = new BlobHttpHeaders(); + } + BlobHeaders.ContentType = blobProperties.ContentType; + BlobHeaders.ContentDisposition = blobProperties.ContentDisposition; + BlobHeaders.ContentEncoding = blobProperties.ContentEncoding; + BlobHeaders.ContentLanguage = blobProperties.ContentLanguage; + BlobHeaders.CacheControl = blobProperties.CacheControl; + BlobHeaders.ContentHash = blobProperties.ContentHash; + } - var signature = _blob.GetSharedAccessSignature(accessPolicy); + private void ReplaceHttpHeaders(BlobDownloadDetails details) + { + if (BlobHeaders == null) + { + BlobHeaders = new BlobHttpHeaders(); + } + BlobHeaders.ContentType = details.ContentType; + BlobHeaders.ContentDisposition = details.ContentDisposition; + BlobHeaders.ContentEncoding = details.ContentEncoding; + BlobHeaders.ContentLanguage = details.ContentLanguage; + BlobHeaders.CacheControl = details.CacheControl; + BlobHeaders.ContentHash = details.ContentHash; + } - return signature; + private void ReplaceHttpHeaders(ResponseHeaders headers) + { + if (BlobHeaders == null) + { + BlobHeaders = new BlobHttpHeaders(); + } + BlobHeaders.ContentType = headers.ContentType; + BlobHeaders.ContentDisposition = headers.TryGetValue(ContentDispositionHeaderName, out var contentDisposition) ? contentDisposition : null; + BlobHeaders.ContentEncoding = headers.TryGetValue(ContentEncodingHeaderName, out var contentEncoding) ? contentEncoding : null; + BlobHeaders.ContentLanguage = headers.TryGetValue(ContentLanguageHeaderName, out var contentLanguage) ? contentLanguage : null; + BlobHeaders.CacheControl = headers.TryGetValue(CacheControlHeaderName, out var cacheControl) ? cacheControl : null; + if (headers.TryGetValue(ContentMd5HeaderName, out var contentHash)) + { + try + { + BlobHeaders.ContentHash = Convert.FromBase64String(contentHash); + } + catch + { + BlobHeaders.ContentHash = null; + } + } } - public async Task StartCopyAsync(ISimpleCloudBlob source, AccessCondition sourceAccessCondition, AccessCondition destAccessCondition) + private void ReplaceMetadata(IDictionary newMetadata) + { + if (Metadata == null) + { + Metadata = new Dictionary(); + } + Metadata.Clear(); + if (newMetadata != null) + { + foreach (var kvp in newMetadata) + { + Metadata.Add(kvp.Key, kvp.Value); + } + } + } + + public async Task GetSharedAccessSignature(FileUriPermissions permissions, DateTimeOffset endOfAccess) + { + var sasBuilder = new BlobSasBuilder + { + BlobContainerName = _blob.BlobContainerName, + BlobName = _blob.Name, + Resource = "b", + StartsOn = DateTimeOffset.UtcNow.AddMinutes(-5), + ExpiresOn = endOfAccess, + }; + sasBuilder.SetPermissions(CloudWrapperHelpers.GetSdkSharedAccessPermissions(permissions)); + + if (_blob.CanGenerateSasUri) + { + // regular SAS + return _blob.GenerateSasUri(sasBuilder).Query; + } + else if (_container?.Account?.UsingTokenCredential == true && _container?.Account?.Client != null) + { + // user delegation SAS + var userDelegationKey = (await _container.Account.Client.GetUserDelegationKeyAsync(sasBuilder.StartsOn, sasBuilder.ExpiresOn)).Value; + var blobUriBuilder = new BlobUriBuilder(_blob.Uri) + { + Sas = sasBuilder.ToSasQueryParameters(userDelegationKey, _blob.AccountName), + }; + return blobUriBuilder.ToUri().Query; + } + else + { + throw new InvalidOperationException("Unsupported blob authentication"); + } + } + + public async Task StartCopyAsync(ISimpleCloudBlob source, IAccessCondition sourceAccessCondition, IAccessCondition destAccessCondition) { // To avoid this we would need to somehow abstract away the primary and secondary storage locations. This // is not worth the effort right now! @@ -169,38 +368,79 @@ public async Task StartCopyAsync(ISimpleCloudBlob source, AccessCondition source throw new ArgumentException($"The source blob must be a {nameof(CloudBlobWrapper)}."); } - await _blob.StartCopyAsync( - sourceWrapper._blob, - sourceAccessCondition: sourceAccessCondition, - destAccessCondition: destAccessCondition, - options: null, - operationContext: null); + // We sort of have 4 cases here: + // 1. sourceWrapper was created using connection string containing account key (shouldn't be the case any longer) + // In this case sourceWrapper._blob.Uri would be a "naked" URI to the blob request to which will fail unless blob is in + // the public container. However, in this case we'd be able to generate SAS URL to use to access it. + // 2. sourceWrapper was created using connection string using SAS token. In this case sourceWrapper._blob.Uri will have + // the same SAS token attached to it automagically (that seems to be Azure.Storage.Blobs feature). + // 3. sourceWrapper uses token credential (MSI or something else provided by Azure.Identity). In this case URI will still + // be naked blob URI. However, assuming destination blob also uses token credential, the underlying implementation + // (in Azure.Storage.Blobs) seem to use destination's token to try to access source and if that gives access, + // everything works. As long as we use the same credential to access both storage accounts (which should be true + // for all our services), it should also work. + // 4. sourceWrapper has BlobSasUri property set (which is indicative of using ICloudBlobClient.GetBlobFromUri with SAS token + // to create the source object). The internal client has the SAS token properly set, but there is no way to fish it out + // so, we assume that property instead contains the appropriate URL that would allow copying from. + // + // If source blob is public none of the above matters. + var sourceUri = sourceWrapper._blob.Uri; + if (sourceWrapper._blob.CanGenerateSasUri) + { + sourceUri = sourceWrapper._blob.GenerateSasUri(BlobSasPermissions.Read, DateTimeOffset.UtcNow.AddMinutes(60)); + } + else if (sourceWrapper.BlobSasUri != null) + { + sourceUri = sourceWrapper.BlobSasUri; + } + + var options = new BlobCopyFromUriOptions + { + SourceConditions = CloudWrapperHelpers.GetSdkAccessCondition(sourceAccessCondition), + DestinationConditions = CloudWrapperHelpers.GetSdkAccessCondition(destAccessCondition), + }; + + var copyOperation = await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + _blob.StartCopyFromUriAsync( + sourceUri, options)); + await FetchAttributesAsync(); } public async Task OpenReadStreamAsync( TimeSpan serverTimeout, - TimeSpan maxExecutionTime, CancellationToken cancellationToken) { - var accessCondition = AccessCondition.GenerateEmptyCondition(); - var blobRequestOptions = new BlobRequestOptions + BlockBlobClient newClient; + BlobClientOptions options = new BlobClientOptions { - ServerTimeout = serverTimeout, - MaximumExecutionTime = maxExecutionTime, - RetryPolicy = new ExponentialRetry(), + Retry = { + NetworkTimeout = serverTimeout, + Mode = Azure.Core.RetryMode.Exponential, + }, }; - var operationContext = new OperationContext(); - - return await _blob.OpenReadAsync(accessCondition, blobRequestOptions, operationContext, cancellationToken); + if (_container?.Account != null) + { + newClient = _container.Account.CreateBlockBlobClient(this, options); + } + else + { + // this might happen if we created blob wrapper from URL, we'll assume authentication + // is built into URI or blob is public. + newClient = new BlockBlobClient(_blob.Uri, options); + } + return await CloudWrapperHelpers.WrapStorageExceptionAsync(() => + newClient.OpenReadAsync(options: null, cancellationToken)); } public async Task DownloadTextIfExistsAsync() { try { - return await _blob.DownloadTextAsync(); + var content = await CloudWrapperHelpers.WrapStorageExceptionAsync(() => _blob.DownloadContentAsync()); + UpdateEtag(content.Value.Details); + return content.Value.Content.ToString(); } - catch (StorageException e) when (IsNotFoundException(e)) + catch (CloudBlobGenericNotFoundException) { return null; } @@ -210,9 +450,9 @@ public async Task FetchAttributesIfExistsAsync() { try { - await _blob.FetchAttributesAsync(); + await FetchAttributesAsync(); } - catch (StorageException e) when (IsNotFoundException(e)) + catch (CloudBlobGenericNotFoundException) { return false; } @@ -225,47 +465,70 @@ public async Task OpenReadIfExistsAsync() { return await OpenReadAsync(accessCondition: null); } - catch (StorageException e) when (IsNotFoundException(e)) + catch (CloudBlobGenericNotFoundException) { return null; } } - private static bool IsNotFoundException(StorageException e) - => ((e.InnerException as WebException)?.Response as HttpWebResponse)?.StatusCode == HttpStatusCode.NotFound; - private static bool IsBlobStorageUri(Uri uri) { return uri.Authority.EndsWith(".blob.core.windows.net"); } - // The default retry policy treats a 304 as an error that requires a retry. We don't want that! - private class DontRetryOnNotModifiedPolicy : IRetryPolicy + private Response UpdateEtag(Response response) { - private IRetryPolicy _innerPolicy; + if (response?.Headers != null) + { + if (response.Headers.ETag.HasValue) + { + _lastSeenEtag = EtagToString(response.Headers.ETag.Value); + } + + ReplaceHttpHeaders(response.Headers); + } + return response; + } - public DontRetryOnNotModifiedPolicy(IRetryPolicy policy) + private Response UpdateEtag(Response propertiesResponse) + { + if (propertiesResponse?.Value != null) { - _innerPolicy = policy; + _lastSeenEtag = EtagToString(propertiesResponse.Value.ETag); } + return propertiesResponse; + } + + private Response UpdateEtag(Response infoResponse) + { + if (infoResponse?.Value != null) + { + _lastSeenEtag = EtagToString(infoResponse.Value.ETag); + } + return infoResponse; + } - public IRetryPolicy CreateInstance() + private Response UpdateEtag(Response infoResponse) + { + if (infoResponse?.Value != null) { - return new DontRetryOnNotModifiedPolicy(_innerPolicy.CreateInstance()); + _lastSeenEtag = EtagToString(infoResponse.Value.ETag); } + return infoResponse; + } - public bool ShouldRetry(int currentRetryCount, int statusCode, Exception lastException, out TimeSpan retryInterval, OperationContext operationContext) + private void UpdateEtag(BlobDownloadDetails details) + { + if (details != null) { - if (statusCode == (int)HttpStatusCode.NotModified) - { - retryInterval = TimeSpan.Zero; - return false; - } - else - { - return _innerPolicy.ShouldRetry(currentRetryCount, statusCode, lastException, out retryInterval, operationContext); - } + _lastSeenEtag = EtagToString(details.ETag); + ReplaceHttpHeaders(details); + ReplaceMetadata(details.Metadata); } } + + // workaround for https://github.com/Azure/azure-sdk-for-net/issues/29942 + private static string EtagToString(ETag etag) + => etag.ToString("H"); } } \ No newline at end of file diff --git a/src/NuGetGallery.Core/Services/CloudWrapperHelpers.cs b/src/NuGetGallery.Core/Services/CloudWrapperHelpers.cs new file mode 100644 index 0000000000..ad9dfc2f64 --- /dev/null +++ b/src/NuGetGallery.Core/Services/CloudWrapperHelpers.cs @@ -0,0 +1,188 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net; +using System.Threading.Tasks; +using Azure; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Sas; + +namespace NuGetGallery +{ + internal static class CloudWrapperHelpers + { + public static BlobTraits GetSdkBlobTraits(ListingDetails listingDetails) + { + BlobTraits traits = BlobTraits.None; + if ((listingDetails & ListingDetails.Metadata) != 0) + { + traits |= BlobTraits.Metadata; + } + if ((listingDetails & ListingDetails.Copy) != 0) + { + traits |= BlobTraits.CopyStatus; + } + return traits; + } + + public static BlobStates GetSdkBlobStates(ListingDetails listingDetails) + { + BlobStates states = BlobStates.None; + if ((listingDetails & ListingDetails.Snapshots) != 0) + { + states |= BlobStates.Snapshots; + } + if ((listingDetails & ListingDetails.UncommittedBlobs) != 0) + { + states |= BlobStates.Uncommitted; + } + if ((listingDetails & ListingDetails.Deleted) != 0) + { + states |= BlobStates.Deleted; + } + return states; + } + + public static CloudBlobCopyStatus GetBlobCopyStatus(CopyStatus? status) + { + if (!status.HasValue) + { + return CloudBlobCopyStatus.None; + } + switch (status.Value) + { + case CopyStatus.Pending: + return CloudBlobCopyStatus.Pending; + case CopyStatus.Success: + return CloudBlobCopyStatus.Success; + case CopyStatus.Aborted: + return CloudBlobCopyStatus.Aborted; + case CopyStatus.Failed: + return CloudBlobCopyStatus.Failed; + default: + throw new ArgumentOutOfRangeException(nameof(status)); + } + } + + public static BlobRequestConditions GetSdkAccessCondition(IAccessCondition accessCondition) + { + if (accessCondition == null) + { + return null; + } + + return new BlobRequestConditions + { + IfMatch = string.IsNullOrEmpty(accessCondition.IfMatchETag) ? (ETag?)null : new Azure.ETag(accessCondition.IfMatchETag), + IfNoneMatch = string.IsNullOrEmpty(accessCondition.IfNoneMatchETag) ? (ETag?)null : new Azure.ETag(accessCondition.IfNoneMatchETag), + }; + } + + public static BlobAccountSasPermissions GetSdkSharedAccessPermissions(FileUriPermissions permissions) + { + BlobAccountSasPermissions convertedPermissions = (BlobAccountSasPermissions)0; + if ((permissions & FileUriPermissions.Read) != 0) + { + convertedPermissions |= BlobAccountSasPermissions.Read; + } + if ((permissions & FileUriPermissions.Write) != 0) + { + convertedPermissions |= BlobAccountSasPermissions.Write; + } + if ((permissions & FileUriPermissions.Delete) != 0) + { + convertedPermissions |= BlobAccountSasPermissions.Delete; + } +#pragma warning disable CS0612 + if ((permissions & FileUriPermissions.List) != 0) +#pragma warning restore CS0612 + { + convertedPermissions |= BlobAccountSasPermissions.List; + } +#pragma warning disable CS0612 + if ((permissions & FileUriPermissions.Add) != 0) +#pragma warning restore CS0612 + { + convertedPermissions |= BlobAccountSasPermissions.Add; + } + if ((permissions & FileUriPermissions.Create) != 0) + { + convertedPermissions |= BlobAccountSasPermissions.Create; + } + return convertedPermissions; + } + + public static async Task WrapStorageExceptionAsync(Func> @delegate) + { + try + { + return await @delegate(); + } + catch (RequestFailedException ex) when (ex.ErrorCode == BlobErrorCode.ContainerNotFound) + { + throw new CloudBlobContainerNotFoundException(ex); + } + catch (RequestFailedException ex) when (ex.ErrorCode == BlobErrorCode.BlobNotFound) + { + throw new CloudBlobNotFoundException(ex); + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.NotFound) + { + throw new CloudBlobGenericNotFoundException(ex); + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict) + { + throw new CloudBlobConflictException(ex); + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.PreconditionFailed) + { + throw new CloudBlobPreconditionFailedException(ex); + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.NotModified) + { + throw new CloudBlobNotModifiedException(ex); + } + catch (RequestFailedException ex) + { + throw new CloudBlobStorageException(ex); + } + } + + public static async Task WrapStorageExceptionAsync(Func @delegate) + { + try + { + await @delegate(); + } + catch (RequestFailedException ex) when (ex.ErrorCode == BlobErrorCode.ContainerNotFound) + { + throw new CloudBlobContainerNotFoundException(ex); + } + catch (RequestFailedException ex) when (ex.ErrorCode == BlobErrorCode.BlobNotFound) + { + throw new CloudBlobNotFoundException(ex); + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.NotFound) + { + throw new CloudBlobGenericNotFoundException(ex); + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict) + { + throw new CloudBlobConflictException(ex); + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.PreconditionFailed) + { + throw new CloudBlobPreconditionFailedException(ex); + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.NotModified) + { + throw new CloudBlobNotModifiedException(ex); + } + catch (RequestFailedException ex) + { + throw new CloudBlobStorageException(ex); + } + } + } +} diff --git a/src/NuGetGallery.Core/Services/ICloudBlobContainer.cs b/src/NuGetGallery.Core/Services/ICloudBlobContainer.cs index d7e75a0577..8da8069b11 100644 --- a/src/NuGetGallery.Core/Services/ICloudBlobContainer.cs +++ b/src/NuGetGallery.Core/Services/ICloudBlobContainer.cs @@ -1,29 +1,27 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Threading; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; namespace NuGetGallery { public interface ICloudBlobContainer { - Task CreateIfNotExistAsync(BlobContainerPermissions permissions); - Task SetPermissionsAsync(BlobContainerPermissions permissions); + Task CreateIfNotExistAsync(bool enablePublicAccess); ISimpleCloudBlob GetBlobReference(string blobAddressUri); - Task ExistsAsync(BlobRequestOptions options, OperationContext operationContext); + Task ExistsAsync(CloudBlobLocationMode? cloudBlobLocationMode); Task DeleteIfExistsAsync(); - Task CreateAsync(BlobContainerPermissions permissions); + Task CreateAsync(bool enablePublicAccess); Task ListBlobsSegmentedAsync( string prefix, bool useFlatBlobListing, - BlobListingDetails blobListingDetails, + ListingDetails blobListingDetails, int? maxResults, - BlobContinuationToken blobContinuationToken, - BlobRequestOptions options, - OperationContext operationContext, + BlobListContinuationToken blobContinuationToken, + TimeSpan? requestTimeout, + CloudBlobLocationMode? cloudBlobLocationMode, CancellationToken cancellationToken); } } diff --git a/src/NuGetGallery.Core/Services/ICloudBlobCopyState.cs b/src/NuGetGallery.Core/Services/ICloudBlobCopyState.cs new file mode 100644 index 0000000000..9bbcd9e962 --- /dev/null +++ b/src/NuGetGallery.Core/Services/ICloudBlobCopyState.cs @@ -0,0 +1,11 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGetGallery +{ + public interface ICloudBlobCopyState + { + CloudBlobCopyStatus Status { get; } + string StatusDescription { get; } + } +} diff --git a/src/NuGetGallery.Core/Services/ICloudBlobProperties.cs b/src/NuGetGallery.Core/Services/ICloudBlobProperties.cs new file mode 100644 index 0000000000..3f96c75051 --- /dev/null +++ b/src/NuGetGallery.Core/Services/ICloudBlobProperties.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGetGallery +{ + public interface ICloudBlobProperties + { + DateTimeOffset? LastModified { get; } + long Length { get; } + string ContentType { get; set; } + string ContentEncoding { get; set; } + string CacheControl { get; set; } + string ContentMD5 { get; } + } +} diff --git a/src/NuGetGallery.Core/Services/ICoreFileStorageService.cs b/src/NuGetGallery.Core/Services/ICoreFileStorageService.cs index 6567652b64..311ff56fd4 100644 --- a/src/NuGetGallery.Core/Services/ICoreFileStorageService.cs +++ b/src/NuGetGallery.Core/Services/ICoreFileStorageService.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage.Blob; namespace NuGetGallery { @@ -54,7 +53,7 @@ public interface ICoreFileStorageService /// The permissions to give to the privileged URI. /// The time when the access ends. /// The URI with privileged access. - Task GetPriviledgedFileUriAsync( + Task GetPrivilegedFileUriAsync( string folderName, string fileName, FileUriPermissions permissions, @@ -155,7 +154,7 @@ Task SetMetadataAsync( Task SetPropertiesAsync( string folderName, string fileName, - Func>, BlobProperties, Task> updatePropertiesAsync); + Func>, ICloudBlobProperties, Task> updatePropertiesAsync); /// /// Returns the etag value for the specified blob. If the blob does not exists it will return null. diff --git a/src/NuGetGallery.Core/Services/ISimpleBlobResultSegment.cs b/src/NuGetGallery.Core/Services/ISimpleBlobResultSegment.cs index 344f06cf23..4e5b2520c0 100644 --- a/src/NuGetGallery.Core/Services/ISimpleBlobResultSegment.cs +++ b/src/NuGetGallery.Core/Services/ISimpleBlobResultSegment.cs @@ -2,13 +2,12 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Collections.Generic; -using Microsoft.WindowsAzure.Storage.Blob; namespace NuGetGallery { public interface ISimpleBlobResultSegment { IReadOnlyList Results { get; } - BlobContinuationToken ContinuationToken { get; } + BlobListContinuationToken ContinuationToken { get; } } } diff --git a/src/NuGetGallery.Core/Services/ISimpleCloudBlob.cs b/src/NuGetGallery.Core/Services/ISimpleCloudBlob.cs index e96bcca960..04272b6417 100644 --- a/src/NuGetGallery.Core/Services/ISimpleCloudBlob.cs +++ b/src/NuGetGallery.Core/Services/ISimpleCloudBlob.cs @@ -6,39 +6,37 @@ using System.IO; using System.Threading; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; namespace NuGetGallery { public interface ISimpleCloudBlob { - BlobProperties Properties { get; } + ICloudBlobProperties Properties { get; } IDictionary Metadata { get; } - CopyState CopyState { get; } + ICloudBlobCopyState CopyState { get; } Uri Uri { get; } string Name { get; } DateTime LastModifiedUtc { get; } string ETag { get; } bool IsSnapshot { get; } - Task OpenReadAsync(AccessCondition accessCondition); - Task OpenWriteAsync(AccessCondition accessCondition); + Task OpenReadAsync(IAccessCondition accessCondition); + Task OpenWriteAsync(IAccessCondition accessCondition, string contentType = null); Task DeleteIfExistsAsync(); Task DownloadToStreamAsync(Stream target); - Task DownloadToStreamAsync(Stream target, AccessCondition accessCondition); + Task DownloadToStreamAsync(Stream target, IAccessCondition accessCondition); Task ExistsAsync(); Task SetPropertiesAsync(); - Task SetPropertiesAsync(AccessCondition accessCondition); - Task SetMetadataAsync(AccessCondition accessCondition); + Task SetPropertiesAsync(IAccessCondition accessCondition); + Task SetMetadataAsync(IAccessCondition accessCondition); Task UploadFromStreamAsync(Stream source, bool overwrite); - Task UploadFromStreamAsync(Stream source, AccessCondition accessCondition); + Task UploadFromStreamAsync(Stream source, IAccessCondition accessCondition); Task FetchAttributesAsync(); - Task StartCopyAsync(ISimpleCloudBlob source, AccessCondition sourceAccessCondition, AccessCondition destAccessCondition); + Task StartCopyAsync(ISimpleCloudBlob source, IAccessCondition sourceAccessCondition, IAccessCondition destAccessCondition); /// /// Generates the shared access signature that if appended to the blob URI @@ -52,7 +50,7 @@ public interface ISimpleCloudBlob /// Null for no time limit. /// /// Shared access signature in form of URI query portion. - string GetSharedAccessSignature(SharedAccessBlobPermissions permissions, DateTimeOffset? endOfAccess); + Task GetSharedAccessSignature(FileUriPermissions permissions, DateTimeOffset endOfAccess); /// /// Opens the seekable read stream to the file in blob storage. @@ -63,7 +61,6 @@ public interface ISimpleCloudBlob /// Read stream for a blob in blob storage. Task OpenReadStreamAsync( TimeSpan serverTimeout, - TimeSpan maxExecutionTime, CancellationToken cancellationToken); Task SnapshotAsync(CancellationToken token); @@ -81,7 +78,7 @@ Task OpenReadStreamAsync( Task FetchAttributesIfExistsAsync(); /// - /// Calls without access condition and returns + /// Calls without access condition and returns /// resulting stream if blob exists. /// /// Stream if the call was successful, null if blob does not exist. diff --git a/src/NuGetGallery.Core/Services/ListingDetails.cs b/src/NuGetGallery.Core/Services/ListingDetails.cs new file mode 100644 index 0000000000..aef2dd6a66 --- /dev/null +++ b/src/NuGetGallery.Core/Services/ListingDetails.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGetGallery +{ + [Flags] + public enum ListingDetails + { + None = 0, + Snapshots = 1, + Metadata = 2, + UncommittedBlobs = 4, + Copy = 8, + Deleted = 0x10, + All = 0x1F + } +} diff --git a/src/NuGetGallery.Core/Services/RevalidationStateService.cs b/src/NuGetGallery.Core/Services/RevalidationStateService.cs index 3fd54f3948..ccdafc4e50 100644 --- a/src/NuGetGallery.Core/Services/RevalidationStateService.cs +++ b/src/NuGetGallery.Core/Services/RevalidationStateService.cs @@ -4,7 +4,6 @@ using System; using System.IO; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; namespace NuGetGallery @@ -72,7 +71,7 @@ public async Task MaybeUpdateStateAsync(Func IsAvailableAsync() { var container = await GetContainerAsync(CoreConstants.Folders.PackagesFolderName); - return await container.ExistsAsync(options: null, operationContext: null); + return await container.ExistsAsync(cloudBlobLocationMode: null); } } } diff --git a/src/NuGetGallery/Services/FileSystemFileStorageService.cs b/src/NuGetGallery/Services/FileSystemFileStorageService.cs index 57adc4098b..2dfa20d3eb 100644 --- a/src/NuGetGallery/Services/FileSystemFileStorageService.cs +++ b/src/NuGetGallery/Services/FileSystemFileStorageService.cs @@ -8,7 +8,6 @@ using System.Threading.Tasks; using System.Web.Hosting; using System.Web.Mvc; -using Microsoft.WindowsAzure.Storage.Blob; using NuGetGallery.Configuration; namespace NuGetGallery @@ -255,7 +254,7 @@ public Task GetFileReadUriAsync(string folderName, string fileName, DateTim throw new NotImplementedException(); } - public Task GetPriviledgedFileUriAsync(string folderName, string fileName, FileUriPermissions permissions, DateTimeOffset endOfAccess) + public Task GetPrivilegedFileUriAsync(string folderName, string fileName, FileUriPermissions permissions, DateTimeOffset endOfAccess) { /// Not implemented for the same reason as . throw new NotImplementedException(); @@ -272,7 +271,7 @@ public Task SetMetadataAsync( public Task SetPropertiesAsync( string folderName, string fileName, - Func>, BlobProperties, Task> updatePropertiesAsync) + Func>, ICloudBlobProperties, Task> updatePropertiesAsync) { return Task.CompletedTask; } diff --git a/src/NuGetGallery/Services/JsonStatisticsService.cs b/src/NuGetGallery/Services/JsonStatisticsService.cs index 5c82fd2c77..f4ae4229c6 100644 --- a/src/NuGetGallery/Services/JsonStatisticsService.cs +++ b/src/NuGetGallery/Services/JsonStatisticsService.cs @@ -7,7 +7,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -384,7 +383,7 @@ public async Task GetPackageDownloadsByVersion(string QuietLog.LogHandledException(e); return null; } - catch (StorageException e) + catch (CloudBlobStorageException e) { QuietLog.LogHandledException(e); return null; @@ -442,7 +441,7 @@ public async Task GetPackageVersionDownloadsByClient(s QuietLog.LogHandledException(e); return null; } - catch (StorageException e) + catch (CloudBlobStorageException e) { QuietLog.LogHandledException(e); return null; diff --git a/src/NuGetGallery/Services/PackageDeleteService.cs b/src/NuGetGallery/Services/PackageDeleteService.cs index adc8945edd..0e52328b8e 100644 --- a/src/NuGetGallery/Services/PackageDeleteService.cs +++ b/src/NuGetGallery/Services/PackageDeleteService.cs @@ -6,7 +6,6 @@ using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; using NuGet.Services.Entities; using NuGet.Versioning; using NuGetGallery.Auditing; @@ -529,7 +528,7 @@ private async Task TryDeleteReadMeMdFile(Package package) await _packageFileService.DeleteReadMeMdFileAsync(package); } } - catch (StorageException) { } + catch (CloudBlobStorageException) { } } private void UnlinkPackageDeprecations(Package package) diff --git a/src/NuGetGallery/Services/StatusService.cs b/src/NuGetGallery/Services/StatusService.cs index 330bd51ce0..d4c31f5f4b 100644 --- a/src/NuGetGallery/Services/StatusService.cs +++ b/src/NuGetGallery/Services/StatusService.cs @@ -10,8 +10,6 @@ using System.Net.Http; using System.Threading.Tasks; using System.Web.Mvc; -using Microsoft.WindowsAzure.Storage.Blob; -using Microsoft.WindowsAzure.Storage.RetryPolicies; using NuGetGallery.Configuration; using NuGetGallery.Helpers; @@ -90,12 +88,11 @@ private bool IsSqlAzureAvailable() try { // Check Storage Availability - BlobRequestOptions options = new BlobRequestOptions(); // Used the LocationMode.SecondaryOnly and not PrimaryThenSecondary for two reasons: // 1. When the primary is down and secondary is up if PrimaryThenSecondary is used there will be an extra and not needed call to the primary. // 2. When the primary is up the secondary status check will return the primary status instead of secondary. - options.LocationMode = _config.ReadOnlyMode ? LocationMode.SecondaryOnly : LocationMode.PrimaryOnly; - var tasks = _cloudStorageAvailabilityChecks.Select(s => s.IsAvailableAsync(options, operationContext : null)); + var locationMode = _config.ReadOnlyMode ? CloudBlobLocationMode.SecondaryOnly : CloudBlobLocationMode.PrimaryOnly; + var tasks = _cloudStorageAvailabilityChecks.Select(s => s.IsAvailableAsync(locationMode)); var eachAvailable = await Task.WhenAll(tasks); storageAvailable = eachAvailable.All(a => a); } diff --git a/src/NuGetGallery/Web.config b/src/NuGetGallery/Web.config index 76e674822d..1b291f9311 100644 --- a/src/NuGetGallery/Web.config +++ b/src/NuGetGallery/Web.config @@ -529,6 +529,18 @@ + + + + + + + + + + + + @@ -649,10 +661,6 @@ - - - - diff --git a/tests/NuGetGallery.Core.Facts/Features/EditableFeatureFlagFileStorageServiceFacts.cs b/tests/NuGetGallery.Core.Facts/Features/EditableFeatureFlagFileStorageServiceFacts.cs index de31ecfa1a..1e030a2c95 100644 --- a/tests/NuGetGallery.Core.Facts/Features/EditableFeatureFlagFileStorageServiceFacts.cs +++ b/tests/NuGetGallery.Core.Facts/Features/EditableFeatureFlagFileStorageServiceFacts.cs @@ -5,11 +5,9 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Net; using System.Text; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage; using Moq; using Newtonsoft.Json; using NuGet.Services.Entities; @@ -621,7 +619,7 @@ public class FactsBase protected readonly Mock _storage; protected readonly Mock _auditing; protected readonly EditableFeatureFlagFileStorageService _target; - protected readonly StorageException _preconditionException; + protected readonly CloudBlobPreconditionFailedException _preconditionException; public FactsBase() { @@ -632,13 +630,7 @@ public FactsBase() _target = new EditableFeatureFlagFileStorageService( _storage.Object, _auditing.Object, logger); - _preconditionException = new StorageException( - new RequestResult - { - HttpStatusCode = (int)HttpStatusCode.PreconditionFailed - }, - "Precondition failed", - new Exception()); + _preconditionException = new CloudBlobPreconditionFailedException(new Exception()); } protected Stream BuildStream(string content) diff --git a/tests/NuGetGallery.Core.Facts/Login/EditableLoginConfigurationFileStorageServiceFacts.cs b/tests/NuGetGallery.Core.Facts/Login/EditableLoginConfigurationFileStorageServiceFacts.cs index 0db037edd1..7cd2038bc8 100644 --- a/tests/NuGetGallery.Core.Facts/Login/EditableLoginConfigurationFileStorageServiceFacts.cs +++ b/tests/NuGetGallery.Core.Facts/Login/EditableLoginConfigurationFileStorageServiceFacts.cs @@ -2,17 +2,13 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Collections.Generic; using System.IO; using System.Linq; -using System.Net; using System.Text; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage; using Moq; using Newtonsoft.Json; -using NuGet.Services.FeatureFlags; using NuGetGallery.Shared; using Xunit; @@ -464,7 +460,7 @@ public class FactsBase { protected readonly Mock _storage; protected readonly EditableLoginConfigurationFileStorageService _target; - protected readonly StorageException _preconditionException; + protected readonly CloudBlobPreconditionFailedException _preconditionException; public FactsBase() { @@ -474,13 +470,7 @@ public FactsBase() _target = new EditableLoginConfigurationFileStorageService( _storage.Object, logger); - _preconditionException = new StorageException( - new RequestResult - { - HttpStatusCode = (int)HttpStatusCode.PreconditionFailed - }, - "Precondition failed", - new Exception()); + _preconditionException = new CloudBlobPreconditionFailedException(new Exception()); } protected Stream BuildStream(string content) { diff --git a/tests/NuGetGallery.Core.Facts/NuGetGallery.Core.Facts.csproj b/tests/NuGetGallery.Core.Facts/NuGetGallery.Core.Facts.csproj index 02a71cc4ca..d46d33450e 100644 --- a/tests/NuGetGallery.Core.Facts/NuGetGallery.Core.Facts.csproj +++ b/tests/NuGetGallery.Core.Facts/NuGetGallery.Core.Facts.csproj @@ -117,7 +117,6 @@ - diff --git a/tests/NuGetGallery.Core.Facts/Services/CloudBlobClientWrapperFacts.cs b/tests/NuGetGallery.Core.Facts/Services/CloudBlobClientWrapperFacts.cs deleted file mode 100644 index 6cabd48cbf..0000000000 --- a/tests/NuGetGallery.Core.Facts/Services/CloudBlobClientWrapperFacts.cs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Reflection; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; -using Xunit; - -namespace NuGetGallery.Services -{ - public class CloudBlobClientWrapperFacts - { - public class GetBlobFromUri - { - [Fact] - public void UsesQueryStringAsSasToken() - { - var blobUrl = "https://example.blob.core.windows.net/packages/nuget.versioning.4.6.0.nupkg"; - var sasToken = "?st=2018-03-12T14%3A55%3A00Z&se=2018-03-13T14%3A55%3A00Z&sp=r&sv=2017-04-17&sr=c&sig=dCXxOlBp6dQHqxTeCRABpr1lfpt40QUaHsAQqs9zHds%3D"; - var uri = new Uri(blobUrl + sasToken); - var target = new CloudBlobClientWrapper("UseDevelopmentStorage=true", readAccessGeoRedundant: false); - - var blob = target.GetBlobFromUri(uri); - - var innerBlob = Assert.IsType(blob - .GetType() - .GetField("_blob", BindingFlags.NonPublic | BindingFlags.Instance) - .GetValue(blob)); - Assert.Equal(AuthenticationScheme.SharedKey, innerBlob.ServiceClient.AuthenticationScheme); - Assert.False(innerBlob.ServiceClient.Credentials.IsAnonymous); - Assert.True(innerBlob.ServiceClient.Credentials.IsSAS); - Assert.False(innerBlob.ServiceClient.Credentials.IsSharedKey); - Assert.Equal(sasToken, innerBlob.ServiceClient.Credentials.SASToken); - Assert.Equal(blobUrl, innerBlob.Uri.AbsoluteUri); - } - - [Fact] - public void UsesAnonymousAuthWhenThereIsNotQueryString() - { - var blobUrl = "https://example.blob.core.windows.net/packages/nuget.versioning.4.6.0.nupkg"; - var uri = new Uri(blobUrl); - var target = new CloudBlobClientWrapper("UseDevelopmentStorage=true", readAccessGeoRedundant: false); - - var blob = target.GetBlobFromUri(uri); - - var innerBlob = Assert.IsType(blob - .GetType() - .GetField("_blob", BindingFlags.NonPublic | BindingFlags.Instance) - .GetValue(blob)); - Assert.Equal(AuthenticationScheme.SharedKey, innerBlob.ServiceClient.AuthenticationScheme); - Assert.True(innerBlob.ServiceClient.Credentials.IsAnonymous); - Assert.False(innerBlob.ServiceClient.Credentials.IsSAS); - Assert.False(innerBlob.ServiceClient.Credentials.IsSharedKey); - Assert.Equal(blobUrl, innerBlob.Uri.AbsoluteUri); - } - } - } -} diff --git a/tests/NuGetGallery.Core.Facts/Services/CloudBlobCoreFileStorageServiceFacts.cs b/tests/NuGetGallery.Core.Facts/Services/CloudBlobCoreFileStorageServiceFacts.cs index a5cbd2277d..b58fbc144a 100644 --- a/tests/NuGetGallery.Core.Facts/Services/CloudBlobCoreFileStorageServiceFacts.cs +++ b/tests/NuGetGallery.Core.Facts/Services/CloudBlobCoreFileStorageServiceFacts.cs @@ -4,12 +4,8 @@ using System; using System.Collections.Generic; using System.IO; -using System.Net; using System.Text; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; -using Microsoft.WindowsAzure.Storage.Blob.Protocol; using Moq; using NuGetGallery.Diagnostics; using Xunit; @@ -52,9 +48,9 @@ public async Task WillCreateABlobContainerForDemandedFoldersIfTheyDoNotExist(str { var fakeBlobClient = new Mock(); var fakeBlobContainer = new Mock(); - fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)).Verifiable(); + fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)).Verifiable(); var simpleCloudBlob = new Mock(); - simpleCloudBlob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())).Returns(Task.FromResult(0)); + simpleCloudBlob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())).Returns(Task.FromResult(0)); fakeBlobContainer.Setup(x => x.GetBlobReference("x.txt")).Returns(simpleCloudBlob.Object); fakeBlobClient.Setup(x => x.GetContainerReference(It.IsAny())).Returns(fakeBlobContainer.Object); @@ -73,9 +69,9 @@ public async Task WillSetPermissionsForDemandedFolderInBlobContainers(string fol var fakeBlobContainer = new Mock(); var simpleCloudBlob = new Mock(); - simpleCloudBlob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())).Returns(Task.FromResult(0)); + simpleCloudBlob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())).Returns(Task.FromResult(0)); - fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)).Verifiable(); + fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)).Verifiable(); fakeBlobContainer.Setup(x => x.GetBlobReference("x.txt")).Returns(simpleCloudBlob.Object); var fakeBlobClient = new Mock(); @@ -111,7 +107,7 @@ public async Task WillGetTheBlobFromTheCorrectFolderContainer(string folderName) { blobContainer = new Mock(); } - blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); return blobContainer.Object; }); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); @@ -134,7 +130,7 @@ public async Task WillDeleteTheBlobIfItExists() fakeBlob.Setup(x => x.DeleteIfExistsAsync()).Returns(Task.FromResult(0)).Verifiable(); fakeBlobClient.Setup(x => x.GetContainerReference(It.IsAny())).Returns(fakeBlobContainer.Object); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); fakeBlob.Setup(x => x.Uri).Returns(new Uri("http://theUri")); var service = CreateService(fakeBlobClient: fakeBlobClient); @@ -200,11 +196,11 @@ public async Task WillDownloadTheFile(string folderName) containerMock = new Mock(); } - containerMock.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + containerMock.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); return containerMock.Object; }); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())).Returns(Task.FromResult(0)).Verifiable(); + fakeBlob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())).Returns(Task.FromResult(0)).Verifiable(); var service = CreateService(fakeBlobClient: fakeBlobClient); await service.GetFileAsync(folderName, "theFileName"); @@ -234,12 +230,12 @@ public async Task WillReturnTheStreamWhenTheFileExists(string folderName) { blobContainer = new Mock(); } - blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); return blobContainer.Object; }); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())) - .Callback((x, _) => { x.WriteByte(42); }) + fakeBlob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())) + .Callback((x, _) => { x.WriteByte(42); }) .Returns(Task.FromResult(0)); var service = CreateService(fakeBlobClient: fakeBlobClient); @@ -269,13 +265,13 @@ public async Task WillReturnNullIfFileDoesNotExist(string folderName) { blobContainer = new Mock(); } - blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); return blobContainer.Object; }); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())).Throws( - new TestableStorageClientException { ErrorCode = BlobErrorCodeStrings.BlobNotFound }); + fakeBlob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())).Throws( + new CloudBlobNotFoundException(null)); var service = CreateService(fakeBlobClient: fakeBlobClient); var stream = await service.GetFileAsync(folderName, "theFileName"); @@ -303,12 +299,12 @@ public async Task WillSetTheStreamPositionToZero(string folderName) { blobContainer = new Mock(); } - blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); return blobContainer.Object; }); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())) - .Callback((x, _) => { x.WriteByte(42); }) + fakeBlob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())) + .Callback((x, _) => { x.WriteByte(42); }) .Returns(Task.FromResult(0)); var service = CreateService(fakeBlobClient: fakeBlobClient); @@ -340,11 +336,11 @@ public async Task WillGetTheBlobFromTheCorrectFolderContainer(string folderName, { blobContainer = new Mock(); } - blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); return blobContainer.Object; }); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlob.Setup(x => x.Properties).Returns(new BlobProperties()); + fakeBlob.Setup(x => x.Properties).Returns(Mock.Of()); fakeBlob.Setup(x => x.Uri).Returns(new Uri("http://theUri")); fakeBlob.Setup(x => x.DeleteIfExistsAsync()).Returns(Task.FromResult(0)); fakeBlob.Setup(x => x.UploadFromStreamAsync(It.IsAny(), true)).Returns(Task.FromResult(0)); @@ -367,8 +363,8 @@ public async Task WillDeleteBlobIfItExistsAndOverwriteTrue() fakeBlob.Setup(x => x.SetPropertiesAsync()).Returns(Task.FromResult(0)).Verifiable(); fakeBlobClient.Setup(x => x.GetContainerReference(It.IsAny())).Returns(fakeBlobContainer.Object); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); - fakeBlob.Setup(x => x.Properties).Returns(new BlobProperties()); + fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + fakeBlob.Setup(x => x.Properties).Returns(Mock.Of()); fakeBlob.Setup(x => x.Uri).Returns(new Uri("http://theUri")); var service = CreateService(fakeBlobClient: fakeBlobClient); @@ -385,14 +381,11 @@ public async Task WillThrowIfBlobExistsAndOverwriteFalse() var fakeBlob = new Mock(); fakeBlob .Setup(x => x.UploadFromStreamAsync(It.IsAny(), false)) - .Throws(new StorageException( - new RequestResult { HttpStatusCode = (int)HttpStatusCode.Conflict }, - "Conflict!", - new Exception("inner"))); + .Throws(new CloudBlobConflictException(new Exception("inner"))); fakeBlobClient.Setup(x => x.GetContainerReference(It.IsAny())).Returns(fakeBlobContainer.Object); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); - fakeBlob.Setup(x => x.Properties).Returns(new BlobProperties()); + fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + fakeBlob.Setup(x => x.Properties).Returns(Mock.Of()); fakeBlob.Setup(x => x.Uri).Returns(new Uri("http://theUri")); var service = CreateService(fakeBlobClient: fakeBlobClient); @@ -406,11 +399,11 @@ public async Task WillUploadThePackageFileToTheBlob() { var fakeBlobClient = new Mock(); var fakeBlobContainer = new Mock(); - fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); var fakeBlob = new Mock(); fakeBlobClient.Setup(x => x.GetContainerReference(It.IsAny())).Returns(fakeBlobContainer.Object); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlob.Setup(x => x.Properties).Returns(new BlobProperties()); + fakeBlob.Setup(x => x.Properties).Returns(Mock.Of()); fakeBlob.Setup(x => x.Uri).Returns(new Uri("http://theUri")); fakeBlob.Setup(x => x.DeleteIfExistsAsync()).Returns(Task.FromResult(0)); fakeBlob.Setup(x => x.SetPropertiesAsync()).Returns(Task.FromResult(0)); @@ -443,11 +436,11 @@ public async Task WillSetTheBlobContentType(string folderName) { blobContainer = new Mock(); } - blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); return blobContainer.Object; }); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlob.Setup(x => x.Properties).Returns(new BlobProperties()); + fakeBlob.Setup(x => x.Properties).Returns(Mock.Of()); fakeBlob.Setup(x => x.Uri).Returns(new Uri("http://theUri")); fakeBlob.Setup(x => x.DeleteIfExistsAsync()).Returns(Task.FromResult(0)); fakeBlob.Setup(x => x.UploadFromStreamAsync(It.IsAny(), true)).Returns(Task.FromResult(0)); @@ -473,11 +466,11 @@ public async Task WillSetTheBlobControlCacheOnPackagesFolder(string folderName) { var fakeBlobClient = new Mock(); var fakeBlobContainer = new Mock(); - fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); var fakeBlob = new Mock(); fakeBlobClient.Setup(x => x.GetContainerReference(It.IsAny())).Returns(fakeBlobContainer.Object); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlob.Setup(x => x.Properties).Returns(new BlobProperties()); + fakeBlob.Setup(x => x.Properties).Returns(Mock.Of()); fakeBlob.Setup(x => x.Uri).Returns(new Uri("http://theUri")); fakeBlob.Setup(x => x.DeleteIfExistsAsync()).Returns(Task.FromResult(0)); fakeBlob.Setup(x => x.SetPropertiesAsync()).Returns(Task.FromResult(0)); @@ -527,11 +520,11 @@ public async Task WillGetTheBlobFromTheCorrectFolderContainer(string folderName, { blobContainer = new Mock(); } - blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); return blobContainer.Object; }); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlob.Setup(x => x.Properties).Returns(new BlobProperties()); + fakeBlob.Setup(x => x.Properties).Returns(Mock.Of()); fakeBlob.Setup(x => x.Uri).Returns(new Uri("http://theUri")); fakeBlob.Setup(x => x.DeleteIfExistsAsync()).Returns(Task.FromResult(0)); fakeBlob.Setup(x => x.UploadFromStreamAsync(It.IsAny(), true)).Returns(Task.FromResult(0)); @@ -555,7 +548,7 @@ public async Task PassesAccessConditionToBlob(IAccessCondition condition, string fakeBlobClient.Setup(x => x.GetContainerReference(It.IsAny())).Returns(fakeBlobContainer.Object); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlob.Setup(x => x.Properties).Returns(new BlobProperties()); + fakeBlob.Setup(x => x.Properties).Returns(Mock.Of()); var service = CreateService(fakeBlobClient: fakeBlobClient); @@ -564,7 +557,7 @@ public async Task PassesAccessConditionToBlob(IAccessCondition condition, string fakeBlob.Verify( b => b.UploadFromStreamAsync( It.IsAny(), - It.Is( + It.Is( c => c.IfMatchETag == expectedIfMatchETag && c.IfNoneMatchETag == expectedIfNoneMatchETag)), Times.Once); } @@ -610,13 +603,10 @@ public async Task ThrowsIfBlobUploadThrowsFileAlreadyExistsException() fakeBlobClient.Setup(x => x.GetContainerReference(It.IsAny())).Returns(fakeBlobContainer.Object); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlob.Setup(x => x.Properties).Returns(new BlobProperties()); + fakeBlob.Setup(x => x.Properties).Returns(Mock.Of()); fakeBlob - .Setup(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny())) - .Throws(new StorageException( - new RequestResult { HttpStatusCode = (int)HttpStatusCode.Conflict }, - "Conflict!", - new Exception("inner"))); + .Setup(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny())) + .Throws(new CloudBlobConflictException(new Exception("inner"))); var service = CreateService(fakeBlobClient: fakeBlobClient); @@ -648,11 +638,11 @@ public async Task WillSetTheBlobContentType(string folderName) { blobContainer = new Mock(); } - blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); return blobContainer.Object; }); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlob.Setup(x => x.Properties).Returns(new BlobProperties()); + fakeBlob.Setup(x => x.Properties).Returns(Mock.Of()); fakeBlob.Setup(x => x.Uri).Returns(new Uri("http://theUri")); fakeBlob.Setup(x => x.DeleteIfExistsAsync()).Returns(Task.FromResult(0)); fakeBlob.Setup(x => x.UploadFromStreamAsync(It.IsAny(), true)).Returns(Task.FromResult(0)); @@ -731,7 +721,7 @@ private static Tuple, Mock, Uri> Setup( } } - public class TheGetPriviledgedFileUriAsyncMethod + public class TheGetPrivilegedFileUriAsyncMethod { private const string folderName = "theFolderName"; private const string fileName = "theFileName"; @@ -742,7 +732,7 @@ public async Task WillThrowIfFolderIsNull() { var service = CreateService(); - var ex = await Assert.ThrowsAsync(() => service.GetPriviledgedFileUriAsync( + var ex = await Assert.ThrowsAsync(() => service.GetPrivilegedFileUriAsync( null, fileName, FileUriPermissions.Read, @@ -755,7 +745,7 @@ public async Task WillThrowIfFilenameIsNull() { var service = CreateService(); - var ex = await Assert.ThrowsAsync(() => service.GetPriviledgedFileUriAsync( + var ex = await Assert.ThrowsAsync(() => service.GetPrivilegedFileUriAsync( folderName, null, FileUriPermissions.Read, @@ -769,7 +759,7 @@ public async Task WillThrowIfEndOfAccessIsInThePast() var service = CreateService(); DateTimeOffset inThePast = DateTimeOffset.UtcNow.AddSeconds(-1); - var ex = await Assert.ThrowsAsync(() => service.GetPriviledgedFileUriAsync( + var ex = await Assert.ThrowsAsync(() => service.GetPrivilegedFileUriAsync( folderName, fileName, FileUriPermissions.Read, @@ -788,11 +778,11 @@ public async Task WillAlwaysUseSasTokenDependingOnContainerAvailability(string c var blobUri = setupResult.Item3; fakeBlob - .Setup(b => b.GetSharedAccessSignature(SharedAccessBlobPermissions.Read, It.IsAny())) - .Returns(signature); + .Setup(b => b.GetSharedAccessSignature(FileUriPermissions.Read, It.IsAny())) + .ReturnsAsync(signature); var service = CreateService(fakeBlobClient); - var uri = await service.GetPriviledgedFileUriAsync( + var uri = await service.GetPrivilegedFileUriAsync( containerName, fileName, FileUriPermissions.Read, @@ -815,14 +805,14 @@ public async Task WillPassTheEndOfAccessTimestampFurther() fakeBlob .Setup(b => b.GetSharedAccessSignature( - SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Delete, + FileUriPermissions.Read | FileUriPermissions.Delete, endOfAccess)) - .Returns(signature) + .ReturnsAsync(signature) .Verifiable(); var service = CreateService(fakeBlobClient); - var uri = await service.GetPriviledgedFileUriAsync( + var uri = await service.GetPrivilegedFileUriAsync( folderName, fileName, FileUriPermissions.Read | FileUriPermissions.Delete, @@ -831,11 +821,11 @@ public async Task WillPassTheEndOfAccessTimestampFurther() string expectedUri = new Uri(blobUri, signature).AbsoluteUri; Assert.Equal(expectedUri, uri.AbsoluteUri); fakeBlob.Verify( - b => b.GetSharedAccessSignature(SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Delete, endOfAccess), + b => b.GetSharedAccessSignature(FileUriPermissions.Read | FileUriPermissions.Delete, endOfAccess), Times.Once); fakeBlob.Verify( - b => b.GetSharedAccessSignature(It.IsAny(), - It.IsAny()), Times.Once); + b => b.GetSharedAccessSignature(It.IsAny(), + It.IsAny()), Times.Once); } private static Tuple, Mock, Uri> Setup(string folderName, string fileName) @@ -911,8 +901,8 @@ public async Task WillUseSasTokenDependingOnContainerAvailability(bool isPublicC var blobUri = setupResult.Item3; fakeBlob - .Setup(b => b.GetSharedAccessSignature(SharedAccessBlobPermissions.Read, It.IsAny())) - .Returns(signature); + .Setup(b => b.GetSharedAccessSignature(FileUriPermissions.Read, It.IsAny())) + .ReturnsAsync(signature); var fakeFolderInformationProvider = new Mock(); fakeFolderInformationProvider .Setup(fip => fip.IsPublicContainer(containerName)) @@ -966,8 +956,8 @@ public async Task WillPassTheEndOfAccessTimestampFurther() var blobUri = setupResult.Item3; fakeBlob - .Setup(b => b.GetSharedAccessSignature(SharedAccessBlobPermissions.Read, endOfAccess)) - .Returns(signature) + .Setup(b => b.GetSharedAccessSignature(FileUriPermissions.Read, endOfAccess)) + .ReturnsAsync(signature) .Verifiable(); var service = CreateService(fakeBlobClient); @@ -976,8 +966,8 @@ public async Task WillPassTheEndOfAccessTimestampFurther() string expectedUri = new Uri(blobUri, signature).AbsoluteUri; Assert.Equal(expectedUri, uri.AbsoluteUri); - fakeBlob.Verify(b => b.GetSharedAccessSignature(SharedAccessBlobPermissions.Read, endOfAccess), Times.Once); - fakeBlob.Verify(b => b.GetSharedAccessSignature(It.IsAny(), It.IsAny()), Times.Once); + fakeBlob.Verify(b => b.GetSharedAccessSignature(FileUriPermissions.Read, endOfAccess), Times.Once); + fakeBlob.Verify(b => b.GetSharedAccessSignature(It.IsAny(), It.IsAny()), Times.Once); } private static Tuple, Mock, Uri> Setup(string folderName, string fileName) @@ -1006,15 +996,15 @@ public class TheCopyFileAsyncMethod private string _srcETag; private Uri _srcUri; private Uri _destUri; - private BlobProperties _srcProperties; + private Mock _srcProperties; private IDictionary _srcMetadata; private string _destFolderName; private string _destFileName; private string _destETag; - private BlobProperties _destProperties; + private Mock _destProperties; private IDictionary _destMetadata; private string _metadataSha512HashAlgorithmId; - private CopyState _destCopyState; + private Mock _destCopyState; private Mock _blobClient; private Mock _srcContainer; private Mock _destContainer; @@ -1028,14 +1018,14 @@ public TheCopyFileAsyncMethod() _srcFileName = "4b6f16cc-7acd-45eb-ac21-33f0d927ec14/nuget.versioning.4.5.0.nupkg"; _srcETag = "\"src-etag\""; _srcUri = new Uri("https://srcexample/srcpackage.nupkg"); - _srcProperties = new BlobProperties(); + _srcProperties = new Mock(); _destFolderName = "packages"; _destFileName = "nuget.versioning.4.5.0.nupkg"; _destETag = "\"dest-etag\""; _destUri = new Uri("https://destexample/destpackage.nupkg"); - _destProperties = new BlobProperties(); - _destCopyState = new CopyState(); - SetDestCopyStatus(CopyStatus.Success); + _destProperties = new Mock(); + _destCopyState = new Mock(); + SetDestCopyStatus(CloudBlobCopyStatus.Success); _metadataSha512HashAlgorithmId = CoreConstants.Sha512HashAlgorithmId; _srcMetadata = new Dictionary(); @@ -1065,7 +1055,7 @@ public TheCopyFileAsyncMethod() .Returns(() => _srcETag); _srcBlobMock .Setup(x => x.Properties) - .Returns(() => _srcProperties); + .Returns(() => _srcProperties.Object); _srcBlobMock .Setup(x => x.Metadata) .Returns(() => _srcMetadata); @@ -1077,10 +1067,10 @@ public TheCopyFileAsyncMethod() .Returns(() => _destETag); _destBlobMock .Setup(x => x.Properties) - .Returns(() => _destProperties); + .Returns(() => _destProperties.Object); _destBlobMock .Setup(x => x.CopyState) - .Returns(() => _destCopyState); + .Returns(() => _destCopyState.Object); _destBlobMock .Setup(x => x.Metadata) .Returns(() => _destMetadata); @@ -1100,11 +1090,11 @@ public async Task WillCopyBlobFromSourceUri() .Returns(_srcBlobMock.Object); _destBlobMock - .Setup(x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(Task.FromResult(0)) - .Callback((_, __, ___) => + .Callback((_, __, ___) => { - SetDestCopyStatus(CopyStatus.Success); + SetDestCopyStatus(CloudBlobCopyStatus.Success); }); // Act @@ -1116,10 +1106,10 @@ await _target.CopyFileAsync( // Assert _destBlobMock.Verify( - x => x.StartCopyAsync(_srcBlobMock.Object, It.IsAny(), It.IsAny()), + x => x.StartCopyAsync(_srcBlobMock.Object, It.IsAny(), It.IsAny()), Times.Once); _destBlobMock.Verify( - x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny()), + x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); _blobClient.Verify( x => x.GetBlobFromUri(_srcUri), @@ -1130,14 +1120,14 @@ await _target.CopyFileAsync( public async Task WillCopyTheFileIfDestinationDoesNotExist() { // Arrange - AccessCondition srcAccessCondition = null; - AccessCondition destAccessCondition = null; + IAccessCondition srcAccessCondition = null; + IAccessCondition destAccessCondition = null; ISimpleCloudBlob srcBlob = null; _destBlobMock - .Setup(x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(Task.FromResult(0)) - .Callback((b, s, d) => + .Callback((b, s, d) => { srcBlob = b; srcAccessCondition = s; @@ -1154,7 +1144,7 @@ await _target.CopyFileAsync( // Assert _destBlobMock.Verify( - x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny()), + x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); Assert.Equal(_srcFileName, srcBlob.Name); Assert.Equal(_srcETag, srcAccessCondition.IfMatchETag); @@ -1166,8 +1156,8 @@ public async Task WillThrowFileAlreadyExistsExceptionForConflict() { // Arrange _destBlobMock - .Setup(x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .Throws(new StorageException(new RequestResult { HttpStatusCode = (int)HttpStatusCode.Conflict }, "Conflict!", inner: null)); + .Setup(x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Throws(new CloudBlobConflictException(null)); // Act & Assert await Assert.ThrowsAsync( @@ -1183,31 +1173,38 @@ await Assert.ThrowsAsync( public async Task WillCopyTheFileIfDestinationHasFailedCopy() { // Arrange - AccessCondition srcAccessCondition = null; - AccessCondition destAccessCondition = null; + IAccessCondition srcAccessCondition = null; + IAccessCondition destAccessCondition = null; ISimpleCloudBlob srcBlob = null; - SetDestCopyStatus(CopyStatus.Failed); + SetDestCopyStatus(CloudBlobCopyStatus.Failed); _destBlobMock - .Setup(x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(Task.FromResult(0)) - .Callback((b, s, d) => + .Callback((b, s, d) => { srcBlob = b; srcAccessCondition = s; destAccessCondition = d; - SetDestCopyStatus(CopyStatus.Pending); + SetDestCopyStatus(CloudBlobCopyStatus.Pending); }); _destBlobMock .Setup(x => x.ExistsAsync()) .ReturnsAsync(true); + var numCalls = 0; _destBlobMock .Setup(x => x.FetchAttributesAsync()) .Returns(Task.FromResult(0)) - .Callback(() => SetDestCopyStatus(CopyStatus.Success)); + .Callback(() => + { + if (++numCalls == 2) + { + SetDestCopyStatus(CloudBlobCopyStatus.Success); + } + }); // Act var srcETag = await _target.CopyFileAsync( @@ -1219,7 +1216,7 @@ public async Task WillCopyTheFileIfDestinationHasFailedCopy() // Assert _destBlobMock.Verify( - x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny()), + x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); Assert.Equal(_srcETag, srcETag); Assert.Equal(_srcFileName, srcBlob.Name); @@ -1231,18 +1228,18 @@ public async Task WillCopyTheFileIfDestinationHasFailedCopy() public async Task WillDefaultToIfNotExists() { // Arrange - AccessCondition srcAccessCondition = null; - AccessCondition destAccessCondition = null; + IAccessCondition srcAccessCondition = null; + IAccessCondition destAccessCondition = null; ISimpleCloudBlob srcBlob = null; _destBlobMock - .Setup(x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(Task.FromResult(0)) - .Callback((b, s, d) => + .Callback((b, s, d) => { srcBlob = b; srcAccessCondition = s; destAccessCondition = d; - SetDestCopyStatus(CopyStatus.Success); + SetDestCopyStatus(CloudBlobCopyStatus.Success); }); // Act @@ -1255,7 +1252,7 @@ await _target.CopyFileAsync( // Assert _destBlobMock.Verify( - x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny()), + x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); Assert.Null(destAccessCondition.IfMatchETag); Assert.Equal("*", destAccessCondition.IfNoneMatchETag); @@ -1265,18 +1262,18 @@ await _target.CopyFileAsync( public async Task UsesProvidedMatchETag() { // Arrange - AccessCondition srcAccessCondition = null; - AccessCondition destAccessCondition = null; + IAccessCondition srcAccessCondition = null; + IAccessCondition destAccessCondition = null; ISimpleCloudBlob srcBlob = null; _destBlobMock - .Setup(x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(Task.FromResult(0)) - .Callback((b, s, d) => + .Callback((b, s, d) => { srcBlob = b; srcAccessCondition = s; destAccessCondition = d; - SetDestCopyStatus(CopyStatus.Success); + SetDestCopyStatus(CloudBlobCopyStatus.Success); }); // Act @@ -1289,7 +1286,7 @@ await _target.CopyFileAsync( // Assert _destBlobMock.Verify( - x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny()), + x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); Assert.Equal("etag!", destAccessCondition.IfMatchETag); Assert.Null(destAccessCondition.IfNoneMatchETag); @@ -1302,7 +1299,7 @@ public async Task NoOpsIfPackageLengthAndHashMatch() SetBlobContentSha512(_srcMetadata, "mwgwUC0MwohHxgMmvQzO7A=="); SetBlobLength(_srcProperties, 42); SetBlobContentSha512(_destMetadata, _srcMetadata[_metadataSha512HashAlgorithmId]); - SetBlobLength(_destProperties, _srcProperties.Length); + SetBlobLength(_destProperties, _srcProperties.Object.Length); _destBlobMock .Setup(x => x.ExistsAsync()) @@ -1318,7 +1315,7 @@ await _target.CopyFileAsync( // Assert _destBlobMock.Verify( - x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny()), + x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); } @@ -1357,7 +1354,7 @@ await _target.CopyFileAsync( // Assert _destBlobMock.Verify( - x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny()), + x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); } @@ -1366,15 +1363,15 @@ public async Task ThrowsIfCopyOperationFails() { // Arrange _destBlobMock - .Setup(x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(x => x.StartCopyAsync(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(Task.FromResult(0)) - .Callback((_, __, ___) => + .Callback((_, __, ___) => { - SetDestCopyStatus(CopyStatus.Failed); + SetDestCopyStatus(CloudBlobCopyStatus.Failed); }); // Act & Assert - var ex = await Assert.ThrowsAsync( + var ex = await Assert.ThrowsAsync( () => _target.CopyFileAsync( _srcFolderName, _srcFileName, @@ -1384,19 +1381,14 @@ public async Task ThrowsIfCopyOperationFails() Assert.Contains("The blob copy operation had copy status Failed", ex.Message); } - private void SetDestCopyStatus(CopyStatus copyStatus) + private void SetDestCopyStatus(CloudBlobCopyStatus copyStatus) { - // We have to use reflection because the setter is not public. - typeof(CopyState) - .GetProperty(nameof(CopyState.Status)) - .SetValue(_destCopyState, copyStatus, null); + _destCopyState.SetupGet(x => x.Status).Returns(copyStatus); } - private void SetBlobLength(BlobProperties properties, long length) + private void SetBlobLength(Mock properties, long length) { - typeof(BlobProperties) - .GetProperty(nameof(BlobProperties.Length)) - .SetValue(properties, length, null); + properties.SetupGet(x => x.Length).Returns(length); } private void SetBlobContentSha512(IDictionary metadata, string contentSha512) @@ -1422,7 +1414,7 @@ public TheSetMetadataAsyncMethod() _blobClient.Setup(x => x.GetContainerReference(It.IsAny())) .Returns(_blobContainer.Object); - _blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())) + _blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())) .Returns(Task.FromResult(0)); _blobContainer.Setup(x => x.GetBlobReference(It.IsAny())) .Returns(_blob.Object); @@ -1433,8 +1425,8 @@ public TheSetMetadataAsyncMethod() [Fact] public async Task WhenLazyStreamRead_ReturnsContent() { - _blob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())) - .Callback((stream, _) => + _blob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())) + .Callback((stream, _) => { using (var writer = new StreamWriter(stream, Encoding.UTF8, bufferSize: 4096, leaveOpen: true)) { @@ -1488,7 +1480,7 @@ public async Task WhenReturnValueIsTrue_MetadataChangesAreNotPersisted() { _blob.SetupGet(x => x.Metadata) .Returns(new Dictionary()); - _blob.Setup(x => x.SetMetadataAsync(It.IsNotNull())) + _blob.Setup(x => x.SetMetadataAsync(It.IsNotNull())) .Returns(Task.FromResult(0)); await _service.SetMetadataAsync( @@ -1524,7 +1516,7 @@ public TheSetPropertiesAsyncMethod() _blobClient.Setup(x => x.GetContainerReference(It.IsAny())) .Returns(_blobContainer.Object); - _blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())) + _blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())) .Returns(Task.FromResult(0)); _blobContainer.Setup(x => x.GetBlobReference(It.IsAny())) .Returns(_blob.Object); @@ -1535,8 +1527,8 @@ public TheSetPropertiesAsyncMethod() [Fact] public async Task WhenLazyStreamRead_ReturnsContent() { - _blob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())) - .Callback((stream, _) => + _blob.Setup(x => x.DownloadToStreamAsync(It.IsAny(), It.IsAny())) + .Callback((stream, _) => { using (var writer = new StreamWriter(stream, Encoding.UTF8, bufferSize: 4096, leaveOpen: true)) { @@ -1568,7 +1560,7 @@ await _service.SetPropertiesAsync( public async Task WhenReturnValueIsFalse_PropertyChangesAreNotPersisted() { _blob.SetupGet(x => x.Properties) - .Returns(new BlobProperties()); + .Returns(Mock.Of()); await _service.SetPropertiesAsync( folderName: CoreConstants.Folders.PackagesFolderName, @@ -1589,8 +1581,8 @@ await _service.SetPropertiesAsync( public async Task WhenReturnValueIsTrue_PropertiesChangesArePersisted() { _blob.SetupGet(x => x.Properties) - .Returns(new BlobProperties()); - _blob.Setup(x => x.SetPropertiesAsync(It.IsNotNull())) + .Returns(Mock.Of()); + _blob.Setup(x => x.SetPropertiesAsync(It.IsNotNull())) .Returns(Task.FromResult(0)); await _service.SetPropertiesAsync( @@ -1626,7 +1618,7 @@ public TheGetETagMethod() _blobClient.Setup(x => x.GetContainerReference(It.IsAny())) .Returns(_blobContainer.Object); - _blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())) + _blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())) .Returns(Task.FromResult(0)); _blobContainer.Setup(x => x.GetBlobReference(It.IsAny())) .Returns(_blob.Object); @@ -1652,7 +1644,7 @@ public async Task VerifyTheETagValue() public async Task VerifyETagIsNullWhenBlobDoesNotExist() { // Arrange - _blob.Setup(x => x.FetchAttributesAsync()).ThrowsAsync(new StorageException("Boo")); + _blob.Setup(x => x.FetchAttributesAsync()).ThrowsAsync(new CloudBlobStorageException("Boo")); // Act var etagValue = await _service.GetETagOrNullAsync(folderName: CoreConstants.Folders.PackagesFolderName, fileName: "a"); diff --git a/tests/NuGetGallery.Core.Facts/Services/CloudBlobCoreFileStorageServiceIntegrationTests.cs b/tests/NuGetGallery.Core.Facts/Services/CloudBlobCoreFileStorageServiceIntegrationTests.cs index 2a98539b81..33628b51ba 100644 --- a/tests/NuGetGallery.Core.Facts/Services/CloudBlobCoreFileStorageServiceIntegrationTests.cs +++ b/tests/NuGetGallery.Core.Facts/Services/CloudBlobCoreFileStorageServiceIntegrationTests.cs @@ -4,14 +4,12 @@ using System; using System.Collections.Concurrent; using System.IO; -using System.Net; using System.Security.Cryptography; using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Auth; -using Microsoft.WindowsAzure.Storage.Blob; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Specialized; using Moq; using NuGetGallery.Diagnostics; using Xunit; @@ -37,8 +35,8 @@ private delegate Task CopyAsync( private readonly string _prefixB; private readonly CloudBlobClientWrapper _clientA; private readonly CloudBlobClientWrapper _clientB; - private readonly CloudBlobClient _blobClientA; - private readonly CloudBlobClient _blobClientB; + private readonly BlobServiceClient _blobClientA; + private readonly BlobServiceClient _blobClientB; private readonly CloudBlobCoreFileStorageService _targetA; private readonly CloudBlobCoreFileStorageService _targetB; @@ -53,15 +51,33 @@ public CloudBlobCoreFileStorageServiceIntegrationTests(BlobStorageFixture fixtur _clientA = new CloudBlobClientWrapper(_fixture.ConnectionStringA, readAccessGeoRedundant: false); _clientB = new CloudBlobClientWrapper(_fixture.ConnectionStringB, readAccessGeoRedundant: false); - _blobClientA = CloudStorageAccount.Parse(_fixture.ConnectionStringA).CreateCloudBlobClient(); - _blobClientB = CloudStorageAccount.Parse(_fixture.ConnectionStringB).CreateCloudBlobClient(); + _blobClientA = new BlobServiceClient(_fixture.ConnectionStringA); + _blobClientB = new BlobServiceClient(_fixture.ConnectionStringB); - var folderInformationProvider = new GalleryCloudBlobContainerInformationProvider(); + var folderInformationProvider = new TestContainerInformationProvider(); _targetA = new CloudBlobCoreFileStorageService(_clientA, Mock.Of(), folderInformationProvider); _targetB = new CloudBlobCoreFileStorageService(_clientB, Mock.Of(), folderInformationProvider); } + private class TestContainerInformationProvider : ICloudBlobContainerInformationProvider + { + public string GetCacheControl(string containerName) + { + return CoreConstants.DefaultCacheControl; + } + + public string GetContentType(string containerName) + { + return CoreConstants.OctetStreamContentType; + } + + public bool IsPublicContainer(string containerName) + { + return false; + } + } + [BlobStorageFact] public async Task EnumeratesBlobs() { @@ -80,20 +96,20 @@ public async Task EnumeratesBlobs() var segmentA = await container.ListBlobsSegmentedAsync( _prefixA, useFlatBlobListing: true, - blobListingDetails: BlobListingDetails.None, + blobListingDetails: ListingDetails.None, maxResults: 2, blobContinuationToken: null, - options: null, - operationContext: null, + requestTimeout: null, + cloudBlobLocationMode: null, cancellationToken: CancellationToken.None); var segmentB = await container.ListBlobsSegmentedAsync( _prefixA, useFlatBlobListing: true, - blobListingDetails: BlobListingDetails.None, + blobListingDetails: ListingDetails.None, maxResults: 2, blobContinuationToken: segmentA.ContinuationToken, - options: null, - operationContext: null, + requestTimeout: null, + cloudBlobLocationMode: null, cancellationToken: CancellationToken.None); // Assert @@ -121,20 +137,20 @@ public async Task EnumeratesSnapshots() var segmentA = await container.ListBlobsSegmentedAsync( _prefixA, useFlatBlobListing: true, - blobListingDetails: BlobListingDetails.Snapshots, + blobListingDetails: ListingDetails.Snapshots, maxResults: 2, blobContinuationToken: null, - options: null, - operationContext: null, + requestTimeout: null, + cloudBlobLocationMode: null, cancellationToken: CancellationToken.None); var segmentB = await container.ListBlobsSegmentedAsync( _prefixA, useFlatBlobListing: true, - blobListingDetails: BlobListingDetails.Snapshots, + blobListingDetails: ListingDetails.Snapshots, maxResults: 2, blobContinuationToken: segmentA.ContinuationToken, - options: null, - operationContext: null, + requestTimeout: null, + cloudBlobLocationMode: null, cancellationToken: CancellationToken.None); // Assert @@ -147,33 +163,6 @@ public async Task EnumeratesSnapshots() Assert.False(segmentB.Results[0].IsSnapshot); } - [BlobStorageFact] - public async Task AllowsDefaultRequestOptionsToBeSet() - { - // Arrange - var folderName = CoreConstants.Folders.ValidationFolderName; - var fileName = _prefixA; - await _targetA.SaveFileAsync( - folderName, - fileName, - new MemoryStream(new byte[1024 * 1024]), - overwrite: false); - var client = new CloudBlobClientWrapper( - _fixture.ConnectionStringA, - new BlobRequestOptions - { - MaximumExecutionTime = TimeSpan.FromMilliseconds(1), - }); - var container = client.GetContainerReference(folderName); - var file = container.GetBlobReference(fileName); - var destination = new MemoryStream(); - - // Act & Assert - // This should throw due to timeout. - var ex = await Assert.ThrowsAsync(() => file.DownloadToStreamAsync(destination)); - Assert.Contains("timeout", ex.Message); - } - [BlobStorageFact] public async Task OpenWriteAsyncReturnsWritableStream() { @@ -210,7 +199,6 @@ public async Task OpenWriteAsyncReturnsWritableStream() Assert.NotNull(file.ETag); Assert.NotEmpty(file.ETag); - Assert.Equal(expectedContentMD5, file.Properties.ContentMD5); } } @@ -265,52 +253,14 @@ await _targetA.SaveFileAsync( var file = container.GetBlobReference(fileName); // Act & Assert - var ex = await Assert.ThrowsAsync( + var ex = await Assert.ThrowsAsync( async () => { - using (var stream = await file.OpenWriteAsync(AccessCondition.GenerateIfNotExistsCondition())) + using (var stream = await file.OpenWriteAsync(AccessConditionWrapper.GenerateIfNotExistsCondition())) { await stream.WriteAsync(Array.Empty(), 0, 0); } }); - Assert.Equal(HttpStatusCode.Conflict, (HttpStatusCode)ex.RequestInformation.HttpStatusCode); - } - - [BlobStorageFact] - public async Task OpenWriteAsyncRejectsETagMismatchFoundAfterUploadStarts() - { - // Arrange - var folderName = CoreConstants.Folders.ValidationFolderName; - var fileName = _prefixA; - var expectedContent = "Hello, world."; - - var container = _clientA.GetContainerReference(folderName); - var file = container.GetBlobReference(fileName); - var writeCount = 0; - - // Act & Assert - var ex = await Assert.ThrowsAsync( - async () => - { - using (var stream = await file.OpenWriteAsync(AccessCondition.GenerateIfNotExistsCondition())) - { - stream.Write(new byte[1], 0, 1); - await stream.FlushAsync(); - writeCount++; - - await _targetA.SaveFileAsync( - folderName, - fileName, - new MemoryStream(Encoding.ASCII.GetBytes(expectedContent)), - overwrite: false); - - stream.Write(new byte[1], 0, 1); - await stream.FlushAsync(); - writeCount++; - } - }); - Assert.Equal(HttpStatusCode.Conflict, (HttpStatusCode)ex.RequestInformation.HttpStatusCode); - Assert.Equal(2, writeCount); } [BlobStorageFact] @@ -329,6 +279,7 @@ await _targetA.SaveFileAsync( var container = _clientA.GetContainerReference(folderName); var file = container.GetBlobReference(fileName); + await file.FetchAttributesAsync(); // Act using (var stream = await file.OpenReadAsync(accessCondition: null)) @@ -355,9 +306,8 @@ public async Task OpenReadAsyncThrowsNotFoundWhenBlobDoesNotExist() // Act & Assert Assert.False(exists); - var ex = await Assert.ThrowsAsync( + var ex = await Assert.ThrowsAsync( () => file.OpenReadAsync(accessCondition: null)); - Assert.Equal(HttpStatusCode.NotFound, (HttpStatusCode)ex.RequestInformation.HttpStatusCode); } [BlobStorageFact] @@ -378,9 +328,8 @@ await _targetA.SaveFileAsync( await file.FetchAttributesAsync(); // Act & Assert - var ex = await Assert.ThrowsAsync( - () => file.OpenReadAsync(accessCondition: AccessCondition.GenerateIfMatchCondition("WON'T MATCH"))); - Assert.Equal(HttpStatusCode.PreconditionFailed, (HttpStatusCode)ex.RequestInformation.HttpStatusCode); + var ex = await Assert.ThrowsAsync( + () => file.OpenReadAsync(accessCondition: AccessConditionWrapper.GenerateIfMatchCondition("WON'T MATCH"))); } [BlobStorageFact] @@ -401,9 +350,8 @@ await _targetA.SaveFileAsync( await file.FetchAttributesAsync(); // Act & Assert - var ex = await Assert.ThrowsAsync( - () => file.OpenReadAsync(accessCondition: AccessCondition.GenerateIfNoneMatchCondition(file.ETag))); - Assert.Equal(HttpStatusCode.NotModified, (HttpStatusCode)ex.RequestInformation.HttpStatusCode); + var ex = await Assert.ThrowsAsync( + () => file.OpenReadAsync(accessCondition: AccessConditionWrapper.GenerateIfNoneMatchCondition(file.ETag))); } [BlobStorageFact] @@ -422,9 +370,10 @@ await _targetA.SaveFileAsync( var container = _clientA.GetContainerReference(folderName); var file = container.GetBlobReference(fileName); + await file.FetchAttributesAsync(); // Act - using (var stream = await file.OpenReadAsync(accessCondition: AccessCondition.GenerateIfNoneMatchCondition("WON'T MATCH"))) + using (var stream = await file.OpenReadAsync(accessCondition: AccessConditionWrapper.GenerateIfNoneMatchCondition("WON'T MATCH"))) using (var streamReader = new StreamReader(stream)) { var actualContent = await streamReader.ReadToEndAsync(); @@ -481,14 +430,14 @@ public async Task ReturnsTheETagMatchingTheContent() Func update = async () => { - var container = _blobClientA.GetContainerReference(folderName); + var container = _blobClientA.GetBlobContainerClient(folderName); for (var i = 1; i <= iterations && !cts.IsCancellationRequested; i++) { - var blob = container.GetBlockBlobReference(fileName); + var blob = container.GetBlockBlobClient(fileName); var content = i.ToString(); - await blob.UploadTextAsync(content); - contentToETag[content] = blob.Properties.ETag; - _output.WriteLine($"Content '{content}' should have etag '{blob.Properties.ETag}'."); + var result = await blob.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes(content))); + contentToETag[content] = result.Value.ETag.ToString("H"); + _output.WriteLine($"Content '{content}' should have etag '{result.Value.ETag.ToString()}'."); } }; @@ -542,22 +491,20 @@ await _targetA.SaveFileAsync( new MemoryStream(Encoding.ASCII.GetBytes(expectedContent)), overwrite: false); - var deleteUri = await _targetA.GetPriviledgedFileUriAsync( + var deleteUri = await _targetA.GetPrivilegedFileUriAsync( folderName, fileName, FileUriPermissions.Read | FileUriPermissions.Delete, DateTimeOffset.UtcNow.AddHours(1)); // Act - var sasToken = new StorageCredentials(deleteUri.Query); - var deleteUriBuilder = new UriBuilder(deleteUri) { Query = null }; - var blob = new CloudBlockBlob(deleteUriBuilder.Uri, sasToken); + var blob = new BlockBlobClient(deleteUri); - var actualContent = await blob.DownloadTextAsync(); + var actualContent = await blob.DownloadContentAsync(); await blob.DeleteAsync(); // Assert - Assert.Equal(expectedContent, actualContent); + Assert.Equal(expectedContent, actualContent.Value.Content.ToString()); var exists = await _targetA.FileExistsAsync(folderName, fileName); Assert.False(exists, "The file should no longer exist."); } @@ -694,9 +641,9 @@ await _targetB.CopyFileAsync( Assert.NotEqual(originalDestETag, finalDestETag); } - private static CloudBlockBlob GetBlob(CloudBlobClient blobClient, string folderName, string fileName) + private static BlockBlobClient GetBlob(BlobServiceClient blobClient, string folderName, string fileName) { - return blobClient.GetContainerReference(folderName).GetBlockBlobReference(fileName); + return blobClient.GetBlobContainerClient(folderName).GetBlockBlobClient(fileName); } private static async Task CopyFileWorksAsync( diff --git a/tests/NuGetGallery.Core.Facts/Services/CloudBlobWrapperFacts.cs b/tests/NuGetGallery.Core.Facts/Services/CloudBlobWrapperFacts.cs index 9671f10e96..29aabff1c7 100644 --- a/tests/NuGetGallery.Core.Facts/Services/CloudBlobWrapperFacts.cs +++ b/tests/NuGetGallery.Core.Facts/Services/CloudBlobWrapperFacts.cs @@ -3,10 +3,12 @@ using System; using System.IO; -using System.Net; +using System.Threading; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized; using Moq; using Xunit; @@ -15,27 +17,22 @@ namespace NuGetGallery.Services public class CloudBlobWrapperFacts { [Fact] - public async Task DownloadsText() + public void UriHasNoQuery() { - var target = new CloudBlobWrapper(_cloudBlobMock.Object); - const string text = "sometext"; - _cloudBlobMock - .Setup(cb => cb.DownloadTextAsync()) - .ReturnsAsync(text) - .Verifiable(); + var client = new CloudBlobClientWrapper("DefaultEndpointsProtocol=https;AccountName=example;SharedAccessSignature=something=somethingelse&sig=somesignature"); + var container = client.GetContainerReference("testcontainer"); + var blob = container.GetBlobReference("testblob"); - var result = await target.DownloadTextIfExistsAsync(); - _cloudBlobMock.VerifyAll(); - Assert.Equal(text, result); + Assert.Empty(blob.Uri.Query); } [Fact] public async Task DownloadTextReturnsNullIfBlobDoesntExist() { - var target = new CloudBlobWrapper(_cloudBlobMock.Object); + var target = new CloudBlobWrapper(_cloudBlobMock.Object, container: null); _cloudBlobMock - .Setup(cb => cb.DownloadTextAsync()) - .ThrowsAsync(CreateBlobNotFoundException()) + .Setup(cb => cb.DownloadContentAsync()) + .ThrowsAsync(new CloudBlobNotFoundException(null)) .Verifiable(); var result = await target.DownloadTextIfExistsAsync(); @@ -45,24 +42,25 @@ public async Task DownloadTextReturnsNullIfBlobDoesntExist() [Fact] public async Task DownloadTextPassesThroughExceptions() { - var target = new CloudBlobWrapper(_cloudBlobMock.Object); + var target = new CloudBlobWrapper(_cloudBlobMock.Object, container: null); var exception = new TestException(); _cloudBlobMock - .Setup(cb => cb.DownloadTextAsync()) + .Setup(cb => cb.DownloadContentAsync()) .ThrowsAsync(exception) .Verifiable(); var thrownException = await Assert.ThrowsAsync(() => target.DownloadTextIfExistsAsync()); Assert.Same(exception, thrownException); } - [Fact] public async Task FetchAttributesIfExistsAsyncReturnsTrueOnSuccess() { - var target = new CloudBlobWrapper(_cloudBlobMock.Object); + var blobProperties = new BlobProperties(); + var response = Response.FromValue(blobProperties, response: null); + var target = new CloudBlobWrapper(_cloudBlobMock.Object, container: null); _cloudBlobMock - .Setup(cb => cb.FetchAttributesAsync()) - .Returns(Task.CompletedTask) + .Setup(cb => cb.GetPropertiesAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(response) .Verifiable(); var result = await target.FetchAttributesIfExistsAsync(); @@ -73,10 +71,10 @@ public async Task FetchAttributesIfExistsAsyncReturnsTrueOnSuccess() [Fact] public async Task FetchAttributesIfExistsAsyncReturnsFalseOnNoBlob() { - var target = new CloudBlobWrapper(_cloudBlobMock.Object); + var target = new CloudBlobWrapper(_cloudBlobMock.Object, container: null); _cloudBlobMock - .Setup(cb => cb.FetchAttributesAsync()) - .ThrowsAsync(CreateBlobNotFoundException()) + .Setup(cb => cb.GetPropertiesAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new CloudBlobNotFoundException(null)) .Verifiable(); var result = await target.FetchAttributesIfExistsAsync(); @@ -87,10 +85,10 @@ public async Task FetchAttributesIfExistsAsyncReturnsFalseOnNoBlob() [Fact] public async Task FetchAttributesIfExistsAsyncPassesThroughExceptions() { - var target = new CloudBlobWrapper(_cloudBlobMock.Object); + var target = new CloudBlobWrapper(_cloudBlobMock.Object, container: null); var exception = new TestException(); _cloudBlobMock - .Setup(cb => cb.FetchAttributesAsync()) + .Setup(cb => cb.GetPropertiesAsync(It.IsAny(), It.IsAny())) .ThrowsAsync(exception) .Verifiable(); @@ -99,30 +97,13 @@ public async Task FetchAttributesIfExistsAsyncPassesThroughExceptions() Assert.Same(exception, thrownException); } - [Fact] - public async Task OpenReadIfExistsAsyncReturnsStream() - { - var target = new CloudBlobWrapper(_cloudBlobMock.Object); - using (var stream = new MemoryStream()) - { - _cloudBlobMock - .Setup(cb => cb.OpenReadAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(stream) - .Verifiable(); - - var returnedStream = await target.OpenReadIfExistsAsync(); - _cloudBlobMock.VerifyAll(); - Assert.Same(stream, returnedStream); - } - } - [Fact] public async Task OpenReadIfExistsAsyncReturnsNullOnNoBlob() { - var target = new CloudBlobWrapper(_cloudBlobMock.Object); + var target = new CloudBlobWrapper(_cloudBlobMock.Object, container: null); _cloudBlobMock - .Setup(cb => cb.OpenReadAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ThrowsAsync(CreateBlobNotFoundException()) + .Setup(cb => cb.DownloadStreamingAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new CloudBlobNotFoundException(null)) .Verifiable(); var returnedStream = await target.OpenReadIfExistsAsync(); @@ -133,10 +114,10 @@ public async Task OpenReadIfExistsAsyncReturnsNullOnNoBlob() [Fact] public async Task OpenReadIfExistsAsyncPassesThroughExceptions() { - var target = new CloudBlobWrapper(_cloudBlobMock.Object); + var target = new CloudBlobWrapper(_cloudBlobMock.Object, container: null); var exception = new TestException(); _cloudBlobMock - .Setup(cb => cb.OpenReadAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(cb => cb.DownloadStreamingAsync(It.IsAny(), It.IsAny())) .ThrowsAsync(exception) .Verifiable(); @@ -147,25 +128,13 @@ public async Task OpenReadIfExistsAsyncPassesThroughExceptions() private class TestException : Exception { - - } - - private static StorageException CreateBlobNotFoundException() - { - var responseMock = new Mock(); - responseMock - .SetupGet(r => r.StatusCode) - .Returns(HttpStatusCode.NotFound); - var innerException = new WebException("inner", null, WebExceptionStatus.Success, responseMock.Object); - var exception = new StorageException("nope", innerException); - return exception; } - private Mock _cloudBlobMock; + private Mock _cloudBlobMock; public CloudBlobWrapperFacts() { - _cloudBlobMock = new Mock(new Uri("https://example.com/blob")); + _cloudBlobMock = new Mock(); } } } diff --git a/tests/NuGetGallery.Core.Facts/Services/RevalidationStateServiceFacts.cs b/tests/NuGetGallery.Core.Facts/Services/RevalidationStateServiceFacts.cs index 9ce25c8040..c6a1f0cd16 100644 --- a/tests/NuGetGallery.Core.Facts/Services/RevalidationStateServiceFacts.cs +++ b/tests/NuGetGallery.Core.Facts/Services/RevalidationStateServiceFacts.cs @@ -5,7 +5,6 @@ using System.IO; using System.Net; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; using Moq; using Newtonsoft.Json; using Xunit; @@ -252,16 +251,9 @@ protected IDisposable Mock( if (storageExceptionCode.HasValue) { - var concurrencyResult = new RequestResult - { - HttpStatusCode = (int)HttpStatusCode.PreconditionFailed - }; - - var concurrencyException = new StorageException(concurrencyResult, "Concurrency exception", inner: null); - _storage .Setup(s => s.SaveFileAsync("revalidation", "state.json", It.IsAny(), It.IsAny())) - .ThrowsAsync(concurrencyException); + .ThrowsAsync(new CloudBlobPreconditionFailedException(null)); } else { diff --git a/tests/NuGetGallery.Facts/Services/CloudBlobFileStorageServiceFacts.cs b/tests/NuGetGallery.Facts/Services/CloudBlobFileStorageServiceFacts.cs index 4cc0e5f0d4..c4df68e673 100644 --- a/tests/NuGetGallery.Facts/Services/CloudBlobFileStorageServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/CloudBlobFileStorageServiceFacts.cs @@ -104,7 +104,7 @@ public async Task WillGetTheBlobFromTheCorrectFolderContainer(string folderName) { blobContainer = new Mock(); } - blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + blobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); return blobContainer.Object; }); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); @@ -127,7 +127,7 @@ public async Task WillReturnARedirectResultToTheBlobUri(string requestUrl, strin var fakeBlob = new Mock(); fakeBlobClient.Setup(x => x.GetContainerReference(It.IsAny())).Returns(fakeBlobContainer.Object); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); var requestUri = new Uri(requestUrl); fakeBlob.Setup(x => x.Uri).Returns(new Uri(requestUri.Scheme + "://theUri")); var service = CreateService(fakeBlobClient: fakeBlobClient); @@ -149,7 +149,7 @@ public async Task WillUseBlobUriPort(string requestUrl, string blobUrl, int expe var fakeBlob = new Mock(); fakeBlobClient.Setup(x => x.GetContainerReference(It.IsAny())).Returns(fakeBlobContainer.Object); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); fakeBlob.Setup(x => x.Uri).Returns(new Uri(blobUrl)); var service = CreateService(fakeBlobClient: fakeBlobClient); @@ -167,7 +167,7 @@ public async Task WillUseISourceDestinationRedirectPolicy() var fakePolicy = new Mock(); fakeBlobClient.Setup(x => x.GetContainerReference(It.IsAny())).Returns(fakeBlobContainer.Object); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); fakeBlob.Setup(x => x.Uri).Returns(new Uri("http://theUri")); fakePolicy.Setup(x => x.IsAllowed(It.IsAny(), It.IsAny())).Returns(true).Verifiable(); var service = CreateService(fakeBlobClient: fakeBlobClient, redirectPolicy: fakePolicy); @@ -188,7 +188,7 @@ public async Task WillThrowIfRedirectIsNotAllowed() var fakePolicy = new Mock(); fakeBlobClient.Setup(x => x.GetContainerReference(It.IsAny())).Returns(fakeBlobContainer.Object); fakeBlobContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(fakeBlob.Object); - fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); + fakeBlobContainer.Setup(x => x.CreateIfNotExistAsync(It.IsAny())).Returns(Task.FromResult(0)); fakeBlob.Setup(x => x.Uri).Returns(new Uri("http://theUri")); fakePolicy.Setup(x => x.IsAllowed(It.IsAny(), It.IsAny())).Returns(false); var service = CreateService(fakeBlobClient: fakeBlobClient, redirectPolicy: fakePolicy); diff --git a/tests/NuGetGallery.Facts/Services/StatusServiceFacts.cs b/tests/NuGetGallery.Facts/Services/StatusServiceFacts.cs index a44337f318..134cd6a7de 100644 --- a/tests/NuGetGallery.Facts/Services/StatusServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/StatusServiceFacts.cs @@ -3,13 +3,10 @@ using System; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; -using Microsoft.WindowsAzure.Storage.RetryPolicies; using Moq; -using Xunit; using NuGetGallery.Auditing; using NuGetGallery.Configuration; +using Xunit; namespace NuGetGallery.Services { @@ -153,15 +150,15 @@ public static ICloudStorageStatusDependency GetCloudStorageStatusDependency(IClo } } - private static void AssertConfigLocationMode(IAppConfiguration config, BlobRequestOptions options) + private static void AssertConfigLocationMode(IAppConfiguration config, CloudBlobLocationMode? locationMode) { if (config.ReadOnlyMode) { - Assert.Equal(LocationMode.SecondaryOnly, options.LocationMode); + Assert.Equal(CloudBlobLocationMode.SecondaryOnly, locationMode.Value); } else { - Assert.Equal(LocationMode.PrimaryOnly, options.LocationMode); + Assert.Equal(CloudBlobLocationMode.PrimaryOnly, locationMode.Value); } } @@ -173,9 +170,9 @@ public CloudStorageStatusDependencyIsAvailable(IAppConfiguration config) _config = config; } - public Task IsAvailableAsync(BlobRequestOptions options, OperationContext operationContext) + public Task IsAvailableAsync(CloudBlobLocationMode? locationMode) { - AssertConfigLocationMode(_config, options); + AssertConfigLocationMode(_config, locationMode); return Task.FromResult(true); } } @@ -189,9 +186,9 @@ public CloudStorageStatusDependencyIsNotAvailable(IAppConfiguration config) _config = config; } - public Task IsAvailableAsync(BlobRequestOptions options, OperationContext operationContext) + public Task IsAvailableAsync(CloudBlobLocationMode? locationMode) { - AssertConfigLocationMode(_config, options); + AssertConfigLocationMode(_config, locationMode); return Task.FromResult(false); } } @@ -205,9 +202,9 @@ public CloudStorageStatusDependencyThrows(IAppConfiguration config) _config = config; } - public async Task IsAvailableAsync(BlobRequestOptions options, OperationContext operationContext) + public async Task IsAvailableAsync(CloudBlobLocationMode? locationMode) { - AssertConfigLocationMode(_config, options); + AssertConfigLocationMode(_config, locationMode); // Just to go async. await Task.Yield(); throw new Exception("Boo");