From a782405fbacf8ac657caec334aae115273a95d02 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Fri, 2 Aug 2024 11:27:17 -0700 Subject: [PATCH] Move to latest ServerCommon version (#10106) * Move to latest ServerCommon version This bring Azure.Data.Tables and Azure.Storage.Blobs into several new places. NuGet.Services.Storage has a new constructor affecting a lot of DI setup. * Don't exclude test helper projects * Handle question mark SAS in more places * Switch to explicit AzureStorageFactory construction This improves compile time validation. --- Directory.Packages.props | 2 +- src/Gallery.CredentialExpiration/Job.cs | 4 +- src/GitHubVulnerabilities2Db/Job.cs | 23 ++++---- src/GitHubVulnerabilities2v3/Job.cs | 23 ++++---- src/Ng/CommandHelpers.cs | 10 +++- src/Ng/Ng.csproj | 2 +- .../DependencyInjectionExtensions.cs | 8 ++- .../Catalog2AzureSearchCommand.cs | 7 ++- .../DetailedReportPostProcessor.cs | 4 +- src/Stats.PostProcessReports/Job.cs | 13 ++--- .../ManualStatusChangeCollectorProcessor.cs | 2 +- .../Container/ContainerWrapper.cs | 10 ++-- .../Container/IContainerWrapper.cs | 3 +- .../IComponentAffectingEntityFactory.cs | 4 +- src/StatusAggregator/Job.cs | 53 +++++++++++++------ .../AddStatusEventManualChangeHandler.cs | 4 +- .../AddStatusMessageManualChangeHandler.cs | 2 +- .../EditStatusEventManualChangeHandler.cs | 2 +- src/StatusAggregator/Table/ITableWrapper.cs | 4 +- src/StatusAggregator/Table/TableWrapper.cs | 36 +++++-------- .../Table/TableWrapperExtensions.cs | 4 +- .../Job.cs | 6 +-- .../Job.cs | 8 +-- test.ps1 | 10 +--- .../BlobStorageVulnerabilityWriterFacts.cs | 11 ++-- .../DetailedReportPostProcessorFacts.cs | 16 +++--- ...nualStatusChangeCollectorProcessorTests.cs | 6 +-- .../Factory/EventFactoryTests.cs | 2 +- .../Factory/IncidentFactoryTests.cs | 2 +- .../Factory/IncidentGroupFactoryTests.cs | 2 +- .../AddStatusEventManualChangeHandlerTests.cs | 2 +- ...ddStatusMessageManualChangeHandlerTests.cs | 4 +- ...EditStatusEventManualChangeHandlerTests.cs | 4 +- .../Messages/MessageFactoryTests.cs | 2 +- .../TestUtility/MockTableWrapperExtensions.cs | 4 +- .../Update/AggregationEntityUpdaterTests.cs | 2 +- .../Update/IncidentUpdaterTests.cs | 2 +- .../Tests.ContextHelpers.csproj | 5 ++ 38 files changed, 172 insertions(+), 136 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 72cf036bf9..36bc0fe6a1 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,7 +4,7 @@ false true 6.9.1 - 2.120.0 + 2.122.0 diff --git a/src/Gallery.CredentialExpiration/Job.cs b/src/Gallery.CredentialExpiration/Job.cs index 3ec707bdb8..5fde1cdb74 100644 --- a/src/Gallery.CredentialExpiration/Job.cs +++ b/src/Gallery.CredentialExpiration/Job.cs @@ -9,12 +9,12 @@ using System.Threading; using System.Threading.Tasks; using Autofac; +using Azure.Storage.Blobs; using Gallery.CredentialExpiration.Models; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; using NuGet.Jobs; using NuGet.Services.Messaging; @@ -52,7 +52,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary { var config = ctx.Resolve(); - return CloudStorageAccount.Parse(config.StorageConnectionString); + var connectionString = AzureStorageFactory.PrepareConnectionString(config.StorageConnectionString); + return new BlobServiceClient(connectionString); }) - .As(); + .As(); containerBuilder - .RegisterType() - .WithParameter( - (parameter, ctx) => parameter.Name == "containerName", - (parameter, ctx) => ctx.Resolve().CursorContainerName) + .Register(ctx => + { + return new AzureStorageFactory( + ctx.Resolve(), + ctx.Resolve().CursorContainerName, + ctx.Resolve>()); + }) .As() .As(); @@ -193,4 +198,4 @@ private DurableCursor CreateCursor(IComponentContext ctx, Func { var config = ctx.Resolve(); - return CloudStorageAccount.Parse(config.StorageConnectionString); + var connectionString = AzureStorageFactory.PrepareConnectionString(config.StorageConnectionString); + return new BlobServiceClient(connectionString); }) - .As(); + .As(); containerBuilder - .RegisterType() - .WithParameter( - (parameter, ctx) => parameter.Name == "containerName", - (parameter, ctx) => ctx.Resolve().V3VulnerabilityContainerName) + .Register(ctx => + { + return new AzureStorageFactory( + ctx.Resolve(), + ctx.Resolve().V3VulnerabilityContainerName, + ctx.Resolve>()); + }) .As() .As(); @@ -145,4 +150,4 @@ private DurableCursor CreateCursor(IComponentContext ctx, Func(argumentNameMap[Arguments.StorageSasValue]); + if (storageSasValue.StartsWith("?")) + { + // workaround for https://github.com/Azure/azure-sdk-for-net/issues/44373 + storageSasValue = storageSasValue.Substring(1); + } + connectionString = $"BlobEndpoint=https://{storageAccountName}.blob.core.windows.net/;SharedAccessSignature={storageSasValue}"; } else @@ -406,4 +412,4 @@ private static QueueServiceClient GetQueueServiceClient(string storageAccountNam } } -} \ No newline at end of file +} diff --git a/src/Ng/Ng.csproj b/src/Ng/Ng.csproj index cd9ba20843..60b3f4730f 100644 --- a/src/Ng/Ng.csproj +++ b/src/Ng/Ng.csproj @@ -20,7 +20,7 @@ - + \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2Registration/DependencyInjectionExtensions.cs b/src/NuGet.Jobs.Catalog2Registration/DependencyInjectionExtensions.cs index 0400025afc..2c62977665 100644 --- a/src/NuGet.Jobs.Catalog2Registration/DependencyInjectionExtensions.cs +++ b/src/NuGet.Jobs.Catalog2Registration/DependencyInjectionExtensions.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -56,7 +56,11 @@ private static void RegisterCursorStorage(ContainerBuilder containerBuilder) .Register(c => { var options = c.Resolve>(); - return new BlobServiceClient(options.Value.StorageConnectionString); + + // workaround for https://github.com/Azure/azure-sdk-for-net/issues/44373 + var connectionString = options.Value.StorageConnectionString.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + + return new BlobServiceClient(connectionString); }) .Keyed(CursorBindingKey); diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchCommand.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchCommand.cs index 6698a6ad14..cb873a4732 100644 --- a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchCommand.cs +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchCommand.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -72,10 +72,13 @@ private async Task ExecuteAsync(CancellationToken token) var frontCursorUri = frontCursorStorage.ResolveUri(CursorRelativeUri); var frontCursor = new DurableCursor(frontCursorUri, frontCursorStorage, DateTime.MinValue); + // workaround for https://github.com/Azure/azure-sdk-for-net/issues/44373 + var connectionString = _options.Value.StorageConnectionString.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + // Log information about where state will be kept. _logger.LogInformation( "Using storage URL: {ContainerUrl}/{StoragePath}", - new BlobServiceClient(_options.Value.StorageConnectionString) + new BlobServiceClient(connectionString) .GetBlobContainerClient(_options.Value.StorageContainer) .Uri .AbsoluteUri, diff --git a/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs b/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs index 391338e13e..8441fa4d2e 100644 --- a/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs +++ b/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs @@ -214,14 +214,14 @@ private async Task CopySourceBlobsAsync(List jsonBlobs, Cancell private async Task> EnumerateSourceBlobsAsync() { - var blobs = await _sourceStorage.List(getMetadata: true, cancellationToken: CancellationToken.None); + var blobs = await _sourceStorage.ListAsync(getMetadata: true, cancellationToken: CancellationToken.None); return blobs.ToList(); } private async Task> EnumerateWorkStorageBlobsAsync() { - var blobs = await _workStorage.List(getMetadata: true, cancellationToken: CancellationToken.None); + var blobs = await _workStorage.ListAsync(getMetadata: true, cancellationToken: CancellationToken.None); return blobs.ToList(); } diff --git a/src/Stats.PostProcessReports/Job.cs b/src/Stats.PostProcessReports/Job.cs index a9b7b53f34..30e4211465 100644 --- a/src/Stats.PostProcessReports/Job.cs +++ b/src/Stats.PostProcessReports/Job.cs @@ -4,11 +4,11 @@ using System.Threading.Tasks; using Autofac; using Autofac.Core; +using Azure.Storage.Blobs; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Microsoft.WindowsAzure.Storage; using NuGet.Jobs; using NuGet.Services.Storage; @@ -39,7 +39,7 @@ protected override void ConfigureAutofacServices(ContainerBuilder containerBuild .Register(c => { var cfg = c.Resolve>().Value; - return CloudStorageAccount.Parse(cfg.StorageAccount); + return new BlobServiceClient(AzureStorageFactory.PrepareConnectionString(cfg.StorageAccount)); }) .AsSelf(); @@ -48,11 +48,10 @@ protected override void ConfigureAutofacServices(ContainerBuilder containerBuild { var cfg = c.Resolve>().Value; var factory = new AzureStorageFactory( - c.Resolve(), + c.Resolve(), cfg.SourceContainerName, c.Resolve>(), cfg.SourcePath + cfg.DetailedReportDirectoryName, - useServerSideCopy: true, initializeContainer: false); var storage = factory.Create(); storage.Verbose = false; @@ -65,11 +64,10 @@ protected override void ConfigureAutofacServices(ContainerBuilder containerBuild { var cfg = c.Resolve>().Value; var factory = new AzureStorageFactory( - c.Resolve(), + c.Resolve(), cfg.WorkContainerName, c.Resolve>(), cfg.WorkPath, - useServerSideCopy: true, initializeContainer: false); var storage = factory.Create(); storage.Verbose = false; @@ -82,11 +80,10 @@ protected override void ConfigureAutofacServices(ContainerBuilder containerBuild { var cfg = c.Resolve>().Value; var factory = new AzureStorageFactory( - c.Resolve(), + c.Resolve(), cfg.DestinationContainerName, c.Resolve>(), cfg.DestinationPath, - useServerSideCopy: true, initializeContainer: false); var storage = factory.Create(); storage.Verbose = false; diff --git a/src/StatusAggregator/Collector/ManualStatusChangeCollectorProcessor.cs b/src/StatusAggregator/Collector/ManualStatusChangeCollectorProcessor.cs index fbe966dc40..4a0eee3e86 100644 --- a/src/StatusAggregator/Collector/ManualStatusChangeCollectorProcessor.cs +++ b/src/StatusAggregator/Collector/ManualStatusChangeCollectorProcessor.cs @@ -59,7 +59,7 @@ public ManualStatusChangeCollectorProcessor( await _handler.Handle(_table, manualChange); } - return manualChanges.Any() ? manualChanges.Max(c => c.Timestamp.UtcDateTime) : (DateTime?)null; + return manualChanges.Any() ? manualChanges.Max(c => c.Timestamp.Value.UtcDateTime) : null; } } } \ No newline at end of file diff --git a/src/StatusAggregator/Container/ContainerWrapper.cs b/src/StatusAggregator/Container/ContainerWrapper.cs index 332a20273b..c97e0df27b 100644 --- a/src/StatusAggregator/Container/ContainerWrapper.cs +++ b/src/StatusAggregator/Container/ContainerWrapper.cs @@ -2,15 +2,15 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage.Blob; +using Azure.Storage.Blobs; namespace StatusAggregator.Container { public class ContainerWrapper : IContainerWrapper { - private readonly CloudBlobContainer _container; + private readonly BlobContainerClient _container; - public ContainerWrapper(CloudBlobContainer container) + public ContainerWrapper(BlobContainerClient container) { _container = container; } @@ -22,8 +22,8 @@ public Task CreateIfNotExistsAsync() public Task SaveBlobAsync(string name, string contents) { - var blob = _container.GetBlockBlobReference(name); - return blob.UploadTextAsync(contents); + var blob = _container.GetBlobClient(name); + return blob.UploadAsync(contents); } } } \ No newline at end of file diff --git a/src/StatusAggregator/Container/IContainerWrapper.cs b/src/StatusAggregator/Container/IContainerWrapper.cs index 68f097cb7f..5aa4cb5500 100644 --- a/src/StatusAggregator/Container/IContainerWrapper.cs +++ b/src/StatusAggregator/Container/IContainerWrapper.cs @@ -2,12 +2,11 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage.Blob; namespace StatusAggregator.Container { /// - /// Simple wrapper for that exists for unit-testing. + /// Simple wrapper for that exists for unit-testing. /// public interface IContainerWrapper { diff --git a/src/StatusAggregator/Factory/IComponentAffectingEntityFactory.cs b/src/StatusAggregator/Factory/IComponentAffectingEntityFactory.cs index 3a246eee43..197fdb348a 100644 --- a/src/StatusAggregator/Factory/IComponentAffectingEntityFactory.cs +++ b/src/StatusAggregator/Factory/IComponentAffectingEntityFactory.cs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage.Table; +using Azure.Data.Tables; using StatusAggregator.Parse; namespace StatusAggregator.Factory @@ -11,7 +11,7 @@ namespace StatusAggregator.Factory /// Creates a given a . /// public interface IComponentAffectingEntityFactory - where TEntity : TableEntity + where TEntity : ITableEntity { Task CreateAsync(ParsedIncident input); } diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs index 0965bd3045..40c0b59801 100644 --- a/src/StatusAggregator/Job.cs +++ b/src/StatusAggregator/Job.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -7,11 +7,12 @@ using System.Threading.Tasks; using Autofac; using Autofac.Core; +using Azure.Data.Tables; +using Azure.Storage.Blobs; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NuGet.Jobs; @@ -126,24 +127,29 @@ private static void AddStorage(ContainerBuilder containerBuilder) var name = statusStorageConnectionBuilder.Name; containerBuilder - .Register(ctx => GetCloudStorageAccount(ctx, statusStorageConnectionBuilder)) - .As() - .Named(name); + .Register(ctx => GetTableServiceClient(ctx, statusStorageConnectionBuilder)) + .As() + .Named(name); containerBuilder .Register(ctx => { - var storageAccount = ctx.ResolveNamed(name); - return GetTableWrapper(ctx, storageAccount); + var tableServiceClient = ctx.ResolveNamed(name); + return GetTableWrapper(ctx, tableServiceClient); }) .As() .Named(name); + containerBuilder + .Register(ctx => GetBlobServiceClient(ctx, statusStorageConnectionBuilder)) + .As() + .Named(name); + containerBuilder .Register(ctx => { - var storageAccount = ctx.ResolveNamed(name); - return GetCloudBlobContainer(ctx, storageAccount); + var blobServiceClient = ctx.ResolveNamed(name); + return GetContainerWrapper(ctx, blobServiceClient); }) .As() .Named(name); @@ -159,23 +165,38 @@ private static void AddStorage(ContainerBuilder containerBuilder) } } - private static CloudStorageAccount GetCloudStorageAccount(IComponentContext ctx, StatusStorageConnectionBuilder statusStorageConnectionBuilder) + private static string GetConnectionString(IComponentContext ctx, StatusStorageConnectionBuilder statusStorageConnectionBuilder) { var configuration = ctx.Resolve(); - return CloudStorageAccount.Parse(statusStorageConnectionBuilder.GetConnectionString(configuration)); + var connectionString = statusStorageConnectionBuilder.GetConnectionString(configuration); + + // workaround for https://github.com/Azure/azure-sdk-for-net/issues/44373 + connectionString = connectionString.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + return connectionString; } - private static ITableWrapper GetTableWrapper(IComponentContext ctx, CloudStorageAccount storageAccount) + private static TableServiceClient GetTableServiceClient(IComponentContext ctx, StatusStorageConnectionBuilder statusStorageConnectionBuilder) + { + string connectionString = GetConnectionString(ctx, statusStorageConnectionBuilder); + return new TableServiceClient(connectionString); + } + + private static ITableWrapper GetTableWrapper(IComponentContext ctx, TableServiceClient tableServiceClient) { var configuration = ctx.Resolve(); - return new TableWrapper(storageAccount, configuration.TableName); + return new TableWrapper(tableServiceClient, configuration.TableName); + } + + private static BlobServiceClient GetBlobServiceClient(IComponentContext ctx, StatusStorageConnectionBuilder statusStorageConnectionBuilder) + { + string connectionString = GetConnectionString(ctx, statusStorageConnectionBuilder); + return new BlobServiceClient(connectionString); } - private static IContainerWrapper GetCloudBlobContainer(IComponentContext ctx, CloudStorageAccount storageAccount) + private static IContainerWrapper GetContainerWrapper(IComponentContext ctx, BlobServiceClient blobServiceClient) { - var blobClient = storageAccount.CreateCloudBlobClient(); var configuration = ctx.Resolve(); - var container = blobClient.GetContainerReference(configuration.ContainerName); + var container = blobServiceClient.GetBlobContainerClient(configuration.ContainerName); return new ContainerWrapper(container); } diff --git a/src/StatusAggregator/Manual/AddStatusEventManualChangeHandler.cs b/src/StatusAggregator/Manual/AddStatusEventManualChangeHandler.cs index d6b2458eb8..a8e4481410 100644 --- a/src/StatusAggregator/Manual/AddStatusEventManualChangeHandler.cs +++ b/src/StatusAggregator/Manual/AddStatusEventManualChangeHandler.cs @@ -22,13 +22,13 @@ public AddStatusEventManualChangeHandler( public async Task Handle(AddStatusEventManualChangeEntity entity) { - var time = entity.Timestamp.UtcDateTime; + var time = entity.Timestamp.Value.UtcDateTime; var eventEntity = new EventEntity( entity.EventAffectedComponentPath, time, affectedComponentStatus: (ComponentStatus)entity.EventAffectedComponentStatus, - endTime: entity.EventIsActive ? (DateTime?)null : time); + endTime: entity.EventIsActive ? null : time); var messageEntity = new MessageEntity( eventEntity, diff --git a/src/StatusAggregator/Manual/AddStatusMessageManualChangeHandler.cs b/src/StatusAggregator/Manual/AddStatusMessageManualChangeHandler.cs index d2b9e0a8eb..89d2384911 100644 --- a/src/StatusAggregator/Manual/AddStatusMessageManualChangeHandler.cs +++ b/src/StatusAggregator/Manual/AddStatusMessageManualChangeHandler.cs @@ -21,7 +21,7 @@ public AddStatusMessageManualChangeHandler( public async Task Handle(AddStatusMessageManualChangeEntity entity) { - var time = entity.Timestamp.UtcDateTime; + var time = entity.Timestamp.Value.UtcDateTime; var eventRowKey = EventEntity.GetRowKey(entity.EventAffectedComponentPath, entity.EventStartTime); var eventEntity = await _table.RetrieveAsync(eventRowKey); diff --git a/src/StatusAggregator/Manual/EditStatusEventManualChangeHandler.cs b/src/StatusAggregator/Manual/EditStatusEventManualChangeHandler.cs index 096dde9e9e..183569a33e 100644 --- a/src/StatusAggregator/Manual/EditStatusEventManualChangeHandler.cs +++ b/src/StatusAggregator/Manual/EditStatusEventManualChangeHandler.cs @@ -29,7 +29,7 @@ public async Task Handle(EditStatusEventManualChangeEntity entity) } eventEntity.AffectedComponentStatus = entity.EventAffectedComponentStatus; - ManualStatusChangeUtility.UpdateEventIsActive(eventEntity, entity.EventIsActive, entity.Timestamp.UtcDateTime); + ManualStatusChangeUtility.UpdateEventIsActive(eventEntity, entity.EventIsActive, entity.Timestamp.Value.UtcDateTime); await _table.ReplaceAsync(eventEntity); } diff --git a/src/StatusAggregator/Table/ITableWrapper.cs b/src/StatusAggregator/Table/ITableWrapper.cs index 180cbef1ac..038ee368e2 100644 --- a/src/StatusAggregator/Table/ITableWrapper.cs +++ b/src/StatusAggregator/Table/ITableWrapper.cs @@ -3,7 +3,7 @@ using System.Linq; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage.Table; +using Azure.Data.Tables; namespace StatusAggregator.Table { @@ -26,6 +26,6 @@ Task RetrieveAsync(string rowKey) Task DeleteAsync(ITableEntity tableEntity); - IQueryable CreateQuery() where T : ITableEntity, new(); + IQueryable CreateQuery() where T : class, ITableEntity, new(); } } diff --git a/src/StatusAggregator/Table/TableWrapper.cs b/src/StatusAggregator/Table/TableWrapper.cs index 253b04a5c2..37bf4ded93 100644 --- a/src/StatusAggregator/Table/TableWrapper.cs +++ b/src/StatusAggregator/Table/TableWrapper.cs @@ -4,8 +4,8 @@ using System; using System.Linq; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Table; +using Azure; +using Azure.Data.Tables; namespace StatusAggregator.Table { @@ -18,16 +18,14 @@ public class TableWrapper : ITableWrapper public const string ETagWildcard = "*"; public TableWrapper( - CloudStorageAccount storageAccount, + TableServiceClient tableServiceClient, string tableName) { - var tableClient = storageAccount?.CreateCloudTableClient() - ?? throw new ArgumentNullException(nameof(storageAccount)); - _table = tableClient.GetTableReference(tableName); + _table = tableServiceClient?.GetTableClient(tableName) ?? throw new ArgumentNullException(nameof(tableServiceClient)); } - private readonly CloudTable _table; + private readonly TableClient _table; public Task CreateIfNotExistsAsync() { @@ -37,49 +35,43 @@ public Task CreateIfNotExistsAsync() public async Task RetrieveAsync(string rowKey) where T : class, ITableEntity { - var operation = TableOperation.Retrieve(TablePartitionKeys.Get(), rowKey); - return (await _table.ExecuteAsync(operation)).Result as T; + return (await _table.GetEntityAsync(TablePartitionKeys.Get(), rowKey)) as T; } public Task InsertAsync(ITableEntity tableEntity) { - return ExecuteOperationAsync(TableOperation.Insert(tableEntity)); + return _table.AddEntityAsync(tableEntity); } public Task InsertOrReplaceAsync(ITableEntity tableEntity) { - return ExecuteOperationAsync(TableOperation.InsertOrReplace(tableEntity)); + return _table.UpsertEntityAsync(tableEntity, TableUpdateMode.Replace); } public Task ReplaceAsync(ITableEntity tableEntity) { - return ExecuteOperationAsync(TableOperation.Replace(tableEntity)); + return _table.UpsertEntityAsync(tableEntity, TableUpdateMode.Replace); } public Task DeleteAsync(string partitionKey, string rowKey) { - return DeleteAsync(partitionKey, rowKey, ETagWildcard); + return _table.DeleteEntityAsync(partitionKey, rowKey); } public Task DeleteAsync(string partitionKey, string rowKey, string eTag) { - return DeleteAsync(new TableEntity(partitionKey, rowKey) { ETag = eTag }); + return _table.DeleteEntityAsync(partitionKey, rowKey, new ETag(eTag)); } public Task DeleteAsync(ITableEntity tableEntity) { - return ExecuteOperationAsync(TableOperation.Delete(tableEntity)); + return _table.DeleteEntityAsync(tableEntity.PartitionKey, tableEntity.RowKey, tableEntity.ETag); } - private Task ExecuteOperationAsync(TableOperation operation) - { - return _table.ExecuteAsync(operation); - } - - public IQueryable CreateQuery() where T : ITableEntity, new() + public IQueryable CreateQuery() where T : class, ITableEntity, new() { return _table - .CreateQuery() + .Query() .AsQueryable() .Where(e => e.PartitionKey == TablePartitionKeys.Get()); } diff --git a/src/StatusAggregator/Table/TableWrapperExtensions.cs b/src/StatusAggregator/Table/TableWrapperExtensions.cs index f067d4446b..26505d6010 100644 --- a/src/StatusAggregator/Table/TableWrapperExtensions.cs +++ b/src/StatusAggregator/Table/TableWrapperExtensions.cs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Linq; -using Microsoft.WindowsAzure.Storage.Table; +using Azure.Data.Tables; using NuGet.Services.Status.Table; namespace StatusAggregator.Table @@ -18,7 +18,7 @@ public static IQueryable GetActiveEntities(this ITableWrapper } public static IQueryable GetChildEntities(this ITableWrapper table, TParent entity) - where TChild : ITableEntity, IChildEntity, new() + where TChild : class, ITableEntity, IChildEntity, new() where TParent : ITableEntity { return table diff --git a/src/Validation.PackageSigning.ProcessSignature/Job.cs b/src/Validation.PackageSigning.ProcessSignature/Job.cs index 9cd97a4435..df9f49842c 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Job.cs +++ b/src/Validation.PackageSigning.ProcessSignature/Job.cs @@ -1,12 +1,12 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using Autofac; +using Azure.Storage.Blobs; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Microsoft.WindowsAzure.Storage; using NuGet.Jobs.Configuration; using NuGet.Jobs.Validation.PackageSigning.Configuration; using NuGet.Jobs.Validation.PackageSigning.Messages; @@ -50,7 +50,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi services.AddTransient(p => { var config = p.GetRequiredService>().Value; - var targetStorageAccount = CloudStorageAccount.Parse(config.DataStorageAccount); + var targetStorageAccount = new BlobServiceClient(AzureStorageFactory.PrepareConnectionString(config.DataStorageAccount)); var storageFactory = new AzureStorageFactory(targetStorageAccount, config.ContainerName, LoggerFactory.CreateLogger()); var storage = storageFactory.Create(); diff --git a/src/Validation.PackageSigning.ValidateCertificate/Job.cs b/src/Validation.PackageSigning.ValidateCertificate/Job.cs index 6caa9eb84e..9dccb01688 100644 --- a/src/Validation.PackageSigning.ValidateCertificate/Job.cs +++ b/src/Validation.PackageSigning.ValidateCertificate/Job.cs @@ -1,12 +1,12 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using Autofac; +using Azure.Storage.Blobs; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Microsoft.WindowsAzure.Storage; using NuGet.Jobs.Validation; using NuGet.Jobs.Validation.PackageSigning.Configuration; using NuGet.Jobs.Validation.PackageSigning.Messages; @@ -31,7 +31,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi services.AddTransient(p => { var config = p.GetRequiredService>().Value; - var targetStorageAccount = CloudStorageAccount.Parse(config.DataStorageAccount); + var targetStorageAccount = new BlobServiceClient(AzureStorageFactory.PrepareConnectionString(config.DataStorageAccount)); var storageFactory = new AzureStorageFactory(targetStorageAccount, config.ContainerName, LoggerFactory.CreateLogger()); var storage = storageFactory.Create(); @@ -50,4 +50,4 @@ protected override void ConfigureAutofacServices(ContainerBuilder containerBuild ConfigureDefaultSubscriptionProcessor(containerBuilder); } } -} \ No newline at end of file +} diff --git a/test.ps1 b/test.ps1 index bedbfbd94d..cd43085361 100644 --- a/test.ps1 +++ b/test.ps1 @@ -31,16 +31,12 @@ $GallerySolution = Join-Path $PSScriptRoot "NuGetGallery.sln" $GalleryProjects = Get-SolutionProjects $GallerySolution $JobsSolution = Join-Path $PSScriptRoot "NuGet.Jobs.sln" $JobsProjects = Get-SolutionProjects $JobsSolution -$ExcludeTestProjects = - "tests\Validation.PackageSigning.Helpers\Tests.ContextHelpers.csproj" Invoke-BuildStep 'Cleaning test results' { Clear-Tests } ` -ev +TestErrors Invoke-BuildStep 'Running gallery tests' { - $GalleryTestProjects = $GalleryProjects ` - | Where-Object { $_.IsTest } ` - | Where-Object { $ExcludeTestProjects -notcontains $_.RelativePath } + $GalleryTestProjects = $GalleryProjects | Where-Object { $_.IsTest } $TestCount = 0 @@ -62,9 +58,7 @@ Invoke-BuildStep 'Running gallery tests' { -ev +TestErrors Invoke-BuildStep 'Running jobs tests' { - $JobsTestProjects = $JobsProjects ` - | Where-Object { $_.IsTest } ` - | Where-Object { $ExcludeTestProjects -notcontains $_.RelativePath } + $JobsTestProjects = $JobsProjects | Where-Object { $_.IsTest } $TestCount = 0 diff --git a/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs b/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs index 3b75941506..12791043e2 100644 --- a/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs +++ b/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -155,9 +155,14 @@ public override Task ExistsAsync(string fileName, CancellationToken cancel return _storage.ExistsAsync(fileName, cancellationToken); } - public override Task> List(bool getMetadata, CancellationToken cancellationToken) + public override IEnumerable List(bool getMetadata) { - return _storage.List(getMetadata, cancellationToken); + return _storage.List(getMetadata); + } + + public override Task> ListAsync(bool getMetadata, CancellationToken cancellationToken) + { + return _storage.ListAsync(getMetadata, cancellationToken); } public override Task SetMetadataAsync(Uri resourceUri, IDictionary metadata) diff --git a/tests/Stats.PostProcessReports.Tests/DetailedReportPostProcessorFacts.cs b/tests/Stats.PostProcessReports.Tests/DetailedReportPostProcessorFacts.cs index 8f36ea9b16..67d77e7f38 100644 --- a/tests/Stats.PostProcessReports.Tests/DetailedReportPostProcessorFacts.cs +++ b/tests/Stats.PostProcessReports.Tests/DetailedReportPostProcessorFacts.cs @@ -33,13 +33,13 @@ public class DetailedReportPostProcessorFacts public async Task DoesntStartIfNoSuccessFile() { _sourceStorageMock - .Setup(ss => ss.List(It.IsAny(), It.IsAny())) - .ReturnsAsync((IEnumerable)new List()); + .Setup(ss => ss.ListAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new List()); await _target.CopyReportsAsync(); _sourceStorageMock - .Verify(ss => ss.List(It.IsAny(), It.IsAny()), Times.Once); + .Verify(ss => ss.ListAsync(It.IsAny(), It.IsAny()), Times.Once); _sourceStorageMock.VerifyNoOtherCalls(); _workStorageMock.VerifyNoOtherCalls(); _destinationStorageMock.VerifyNoOtherCalls(); @@ -186,8 +186,8 @@ public async Task SkipsProcessedFiles() { "FilesCreated", "123" } }; _workStorageMock - .Setup(ss => ss.List(It.IsAny(), It.IsAny())) - .ReturnsAsync(() => (IEnumerable)new List(_workFiles.Select(f => Blob( + .Setup(ss => ss.ListAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => new List(_workFiles.Select(f => Blob( _workStorageMock, f, f == file1 ? file1Metadata : null)))); @@ -278,7 +278,7 @@ private static void SetupStorageMock(Mock mock, string baseUrl, Func s.Load(It.IsAny(), It.IsAny())) - .ReturnsAsync((StorageContent)new StringStorageContent("")); + .ReturnsAsync(new StringStorageContent("")); mock .Setup(s => s.ExistsAsync(It.IsAny(), It.IsAny())) .ReturnsAsync((string filename, CancellationToken _) => @@ -286,8 +286,8 @@ private static void SetupStorageMock(Mock mock, string baseUrl, Func s.List(It.IsAny(), It.IsAny())) - .ReturnsAsync(() => (IEnumerable)new List(files().Select(f => Blob(mock, f)))); + .Setup(s => s.ListAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => new List(files().Select(f => Blob(mock, f)))); mock .Setup(s => s.ResolveUri(It.IsAny())) .Returns((string f) => Blob(mock, f).Uri); diff --git a/tests/StatusAggregator.Tests/Collector/ManualStatusChangeCollectorProcessorTests.cs b/tests/StatusAggregator.Tests/Collector/ManualStatusChangeCollectorProcessorTests.cs index 0ea4e0071f..5388b82b59 100644 --- a/tests/StatusAggregator.Tests/Collector/ManualStatusChangeCollectorProcessorTests.cs +++ b/tests/StatusAggregator.Tests/Collector/ManualStatusChangeCollectorProcessorTests.cs @@ -67,14 +67,14 @@ public async Task HandlesManualChanges() .Returns(Task.CompletedTask) .Callback((table, entity) => { - var nextTimestamp = entity.Timestamp; + var nextTimestamp = entity.Timestamp.Value; Assert.True(nextTimestamp > lastTimestamp); lastTimestamp = nextTimestamp; }); var result = await Processor.FetchSince(Cursor); - Assert.Equal(secondChange.Timestamp.UtcDateTime, result); + Assert.Equal(secondChange.Timestamp.Value.UtcDateTime, result); Handler .Verify( @@ -109,7 +109,7 @@ public async Task DoesNotFilterByCursor() var result = await Processor.FetchSince(Cursor); - Assert.Equal(change.Timestamp.UtcDateTime, result); + Assert.Equal(change.Timestamp.Value.UtcDateTime, result); Handler.Verify(); } diff --git a/tests/StatusAggregator.Tests/Factory/EventFactoryTests.cs b/tests/StatusAggregator.Tests/Factory/EventFactoryTests.cs index f6babae5c6..e7fb3e8d90 100644 --- a/tests/StatusAggregator.Tests/Factory/EventFactoryTests.cs +++ b/tests/StatusAggregator.Tests/Factory/EventFactoryTests.cs @@ -3,8 +3,8 @@ using System; using System.Threading.Tasks; +using Azure.Data.Tables; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage.Table; using Moq; using NuGet.Services.Incidents; using NuGet.Services.Status; diff --git a/tests/StatusAggregator.Tests/Factory/IncidentFactoryTests.cs b/tests/StatusAggregator.Tests/Factory/IncidentFactoryTests.cs index d474dcb17c..71b0cb61d7 100644 --- a/tests/StatusAggregator.Tests/Factory/IncidentFactoryTests.cs +++ b/tests/StatusAggregator.Tests/Factory/IncidentFactoryTests.cs @@ -3,8 +3,8 @@ using System; using System.Threading.Tasks; +using Azure.Data.Tables; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage.Table; using Moq; using NuGet.Services.Incidents; using NuGet.Services.Status; diff --git a/tests/StatusAggregator.Tests/Factory/IncidentGroupFactoryTests.cs b/tests/StatusAggregator.Tests/Factory/IncidentGroupFactoryTests.cs index 459d0bb25a..be7548e071 100644 --- a/tests/StatusAggregator.Tests/Factory/IncidentGroupFactoryTests.cs +++ b/tests/StatusAggregator.Tests/Factory/IncidentGroupFactoryTests.cs @@ -3,8 +3,8 @@ using System; using System.Threading.Tasks; +using Azure.Data.Tables; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage.Table; using Moq; using NuGet.Services.Incidents; using NuGet.Services.Status; diff --git a/tests/StatusAggregator.Tests/Manual/AddStatusEventManualChangeHandlerTests.cs b/tests/StatusAggregator.Tests/Manual/AddStatusEventManualChangeHandlerTests.cs index c676d57568..15770b266b 100644 --- a/tests/StatusAggregator.Tests/Manual/AddStatusEventManualChangeHandlerTests.cs +++ b/tests/StatusAggregator.Tests/Manual/AddStatusEventManualChangeHandlerTests.cs @@ -36,7 +36,7 @@ public async Task SavesMessageAndEvent(bool eventIsActive) Timestamp = new DateTimeOffset(2018, 8, 21, 0, 0, 0, TimeSpan.Zero) }; - var time = entity.Timestamp.UtcDateTime; + var time = entity.Timestamp.Value.UtcDateTime; var eventRowKey = EventEntity.GetRowKey(entity.EventAffectedComponentPath, time); _table.Setup(x => x.InsertAsync( diff --git a/tests/StatusAggregator.Tests/Manual/AddStatusMessageManualChangeHandlerTests.cs b/tests/StatusAggregator.Tests/Manual/AddStatusMessageManualChangeHandlerTests.cs index 9c21b10c66..63e5428712 100644 --- a/tests/StatusAggregator.Tests/Manual/AddStatusMessageManualChangeHandlerTests.cs +++ b/tests/StatusAggregator.Tests/Manual/AddStatusMessageManualChangeHandlerTests.cs @@ -35,7 +35,7 @@ public async Task DoesNotSaveIfEventIsMissing() Timestamp = new DateTimeOffset(2018, 8, 21, 0, 0, 0, TimeSpan.Zero) }; - var time = entity.Timestamp.UtcDateTime; + var time = entity.Timestamp.Value.UtcDateTime; var eventRowKey = EventEntity.GetRowKey(entity.EventAffectedComponentPath, entity.EventStartTime); _table @@ -80,7 +80,7 @@ public async Task SavesNewMessageAndUpdatesEventIfNecessary(bool eventIsActive, Timestamp = new DateTimeOffset(2018, 8, 21, 0, 0, 0, TimeSpan.Zero) }; - var time = entity.Timestamp.UtcDateTime; + var time = entity.Timestamp.Value.UtcDateTime; var existingEntity = new EventEntity( entity.EventAffectedComponentPath, diff --git a/tests/StatusAggregator.Tests/Manual/EditStatusEventManualChangeHandlerTests.cs b/tests/StatusAggregator.Tests/Manual/EditStatusEventManualChangeHandlerTests.cs index 736a0104db..70ae31174b 100644 --- a/tests/StatusAggregator.Tests/Manual/EditStatusEventManualChangeHandlerTests.cs +++ b/tests/StatusAggregator.Tests/Manual/EditStatusEventManualChangeHandlerTests.cs @@ -35,7 +35,7 @@ public async Task ThrowsArgumentExceptionIfMissingEvent() Timestamp = new DateTimeOffset(2018, 8, 21, 0, 0, 0, TimeSpan.Zero) }; - var time = entity.Timestamp.UtcDateTime; + var time = entity.Timestamp.Value.UtcDateTime; var eventRowKey = EventEntity.GetRowKey(entity.EventAffectedComponentPath, entity.EventStartTime); _table @@ -68,7 +68,7 @@ public async Task EditsEvent(bool eventIsActive, bool shouldEventBeActive) Timestamp = new DateTimeOffset(2018, 8, 21, 0, 0, 0, TimeSpan.Zero) }; - var time = entity.Timestamp.UtcDateTime; + var time = entity.Timestamp.Value.UtcDateTime; var eventRowKey = EventEntity.GetRowKey(entity.EventAffectedComponentPath, entity.EventStartTime); var existingEntity = diff --git a/tests/StatusAggregator.Tests/Messages/MessageFactoryTests.cs b/tests/StatusAggregator.Tests/Messages/MessageFactoryTests.cs index f17dc599de..ac74c800ae 100644 --- a/tests/StatusAggregator.Tests/Messages/MessageFactoryTests.cs +++ b/tests/StatusAggregator.Tests/Messages/MessageFactoryTests.cs @@ -3,8 +3,8 @@ using System; using System.Threading.Tasks; +using Azure.Data.Tables; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage.Table; using Moq; using NuGet.Services.Status; using NuGet.Services.Status.Table; diff --git a/tests/StatusAggregator.Tests/TestUtility/MockTableWrapperExtensions.cs b/tests/StatusAggregator.Tests/TestUtility/MockTableWrapperExtensions.cs index 2f0098f228..b380fc4658 100644 --- a/tests/StatusAggregator.Tests/TestUtility/MockTableWrapperExtensions.cs +++ b/tests/StatusAggregator.Tests/TestUtility/MockTableWrapperExtensions.cs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Linq; -using Microsoft.WindowsAzure.Storage.Table; +using Azure.Data.Tables; using Moq; using StatusAggregator.Table; @@ -13,7 +13,7 @@ public static class MockTableWrapperExtensions public static void SetupQuery( this Mock mock, params T[] results) - where T : ITableEntity, new() + where T : class, ITableEntity, new() { mock .Setup(x => x.CreateQuery()) diff --git a/tests/StatusAggregator.Tests/Update/AggregationEntityUpdaterTests.cs b/tests/StatusAggregator.Tests/Update/AggregationEntityUpdaterTests.cs index 8f1ba896e7..279dde04e5 100644 --- a/tests/StatusAggregator.Tests/Update/AggregationEntityUpdaterTests.cs +++ b/tests/StatusAggregator.Tests/Update/AggregationEntityUpdaterTests.cs @@ -3,8 +3,8 @@ using System; using System.Threading.Tasks; +using Azure.Data.Tables; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage.Table; using Moq; using NuGet.Services.Status.Table; using StatusAggregator.Table; diff --git a/tests/StatusAggregator.Tests/Update/IncidentUpdaterTests.cs b/tests/StatusAggregator.Tests/Update/IncidentUpdaterTests.cs index 7c6150dfc3..7d6f9a707e 100644 --- a/tests/StatusAggregator.Tests/Update/IncidentUpdaterTests.cs +++ b/tests/StatusAggregator.Tests/Update/IncidentUpdaterTests.cs @@ -4,8 +4,8 @@ using System; using System.Net; using System.Threading.Tasks; +using Azure.Data.Tables; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage.Table; using Moq; using NuGet.Services.Incidents; using NuGet.Services.Status.Table; diff --git a/tests/Validation.PackageSigning.Helpers/Tests.ContextHelpers.csproj b/tests/Validation.PackageSigning.Helpers/Tests.ContextHelpers.csproj index 097aedb2ed..b164732d23 100644 --- a/tests/Validation.PackageSigning.Helpers/Tests.ContextHelpers.csproj +++ b/tests/Validation.PackageSigning.Helpers/Tests.ContextHelpers.csproj @@ -7,6 +7,11 @@ + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all +