Skip to content

Commit

Permalink
Merge branch 'agr-stsdk-ft-advay26' into dev-feature-sdkmigration
Browse files Browse the repository at this point in the history
  • Loading branch information
drewgillies committed Aug 22, 2024
2 parents b3855a8 + 8213c64 commit 774816d
Show file tree
Hide file tree
Showing 16 changed files with 258 additions and 83 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,8 @@ public static ContainerBuilder AddCatalog2Registration(this ContainerBuilder con
RegisterCursorStorage(containerBuilder);

containerBuilder
.Register<ICloudBlobClient>(c =>
{
var options = c.Resolve<IOptionsSnapshot<Catalog2RegistrationConfiguration>>();
return new CloudBlobClientWrapper(
options.Value.StorageConnectionString,
requestTimeout: DefaultBlobRequestOptions.ServerTimeout);
});
.RegisterStorageAccount<Catalog2RegistrationConfiguration>(c => c.StorageConnectionString, requestTimeout: DefaultBlobRequestOptions.ServerTimeout)
.As<ICloudBlobClient>();

containerBuilder.Register(c => new Catalog2RegistrationCommand(
c.Resolve<ICollector>(),
Expand Down
9 changes: 2 additions & 7 deletions src/NuGet.Jobs.Common/JsonConfigurationJob.cs
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ protected virtual void ConfigureDefaultJobServices(IServiceCollection services,
services.Configure<ValidationDbConfiguration>(configurationRoot.GetSection(ValidationDbConfigurationSectionName));
services.Configure<ServiceBusConfiguration>(configurationRoot.GetSection(ServiceBusConfigurationSectionName));
services.Configure<ValidationStorageConfiguration>(configurationRoot.GetSection(ValidationStorageConfigurationSectionName));
services.ConfigureStorageMsi(configurationRoot);

services.AddSingleton(new TelemetryClient(ApplicationInsightsConfiguration.TelemetryConfiguration));
services.AddTransient<ITelemetryClient, TelemetryClientWrapper>();
Expand Down Expand Up @@ -197,13 +198,7 @@ private void AddScopedSqlConnectionFactory<TDbConfiguration>(IServiceCollection
public static void ConfigureFeatureFlagAutofacServices(ContainerBuilder containerBuilder)
{
containerBuilder
.Register(c =>
{
var options = c.Resolve<IOptionsSnapshot<FeatureFlagConfiguration>>();
return new CloudBlobClientWrapper(
options.Value.ConnectionString,
requestTimeout: TimeSpan.FromMinutes(2));
})
.RegisterStorageAccount<FeatureFlagConfiguration>(c => c.ConnectionString, requestTimeout: TimeSpan.FromMinutes(2))
.Keyed<ICloudBlobClient>(FeatureFlagBindingKey);

containerBuilder
Expand Down
135 changes: 135 additions & 0 deletions src/NuGet.Jobs.Common/StorageAccountExtensions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using Autofac;
using Autofac.Builder;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using NuGet.Services.Configuration;
using NuGetGallery;

namespace NuGet.Jobs
{
public static class StorageAccountHelper
{
public static IServiceCollection ConfigureStorageMsi(
this IServiceCollection serviceCollection,
IConfiguration configuration,
string storageUseManagedIdentityPropertyName = null,
string storageManagedIdentityClientIdPropertyName = null)
{
if (serviceCollection == null)
{
throw new ArgumentNullException(nameof(serviceCollection));
}
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}

storageUseManagedIdentityPropertyName ??= Constants.StorageUseManagedIdentityPropertyName;
storageManagedIdentityClientIdPropertyName ??= Constants.StorageManagedIdentityClientIdPropertyName;

string useManagedIdentityStr = configuration[storageUseManagedIdentityPropertyName];
string managedIdentityClientId = string.IsNullOrEmpty(configuration[storageManagedIdentityClientIdPropertyName])
? configuration[Constants.ManagedIdentityClientIdKey]
: configuration[storageManagedIdentityClientIdPropertyName];
bool useManagedIdentity = false;
if (!string.IsNullOrWhiteSpace(useManagedIdentityStr))
{
useManagedIdentity = bool.Parse(useManagedIdentityStr);
}
return serviceCollection.Configure<StorageMsiConfiguration>(storageConfiguration =>
{
storageConfiguration.UseManagedIdentity = useManagedIdentity;
storageConfiguration.ManagedIdentityClientId = managedIdentityClientId;
});
}

public static CloudBlobClientWrapper CreateCloudBlobClient(
this IServiceProvider serviceProvider,
string storageConnectionString,
bool readAccessGeoRedundant = false,
TimeSpan? requestTimeout = null)
{
if (serviceProvider == null)
{
throw new ArgumentNullException(nameof(serviceProvider));
}
if (string.IsNullOrWhiteSpace(storageConnectionString))
{
throw new ArgumentException($"{nameof(storageConnectionString)} cannot be null or empty.", nameof(storageConnectionString));
}

var msiConfiguration = serviceProvider.GetRequiredService<IOptions<StorageMsiConfiguration>>().Value;
return CreateCloudBlobClient(
msiConfiguration,
storageConnectionString,
readAccessGeoRedundant,
requestTimeout);
}

public static IRegistrationBuilder<CloudBlobClientWrapper, SimpleActivatorData, SingleRegistrationStyle> RegisterStorageAccount<TConfiguration>(
this ContainerBuilder builder,
Func<TConfiguration, string> getConnectionString,
Func<TConfiguration, bool> getReadAccessGeoRedundant = null,
TimeSpan? requestTimeout = null)
where TConfiguration : class, new()
{
if (builder == null)
{
throw new ArgumentNullException(nameof(builder));
}
if (getConnectionString == null)
{
throw new ArgumentNullException(nameof(getConnectionString));
}

return builder.Register(c =>
{
var options = c.Resolve<IOptionsSnapshot<TConfiguration>>();
string storageConnectionString = getConnectionString(options.Value);
bool readAccessGeoRedundant = getReadAccessGeoRedundant?.Invoke(options.Value) ?? false;
var msiConfiguration = c.Resolve<IOptions<StorageMsiConfiguration>>().Value;
return CreateCloudBlobClient(
msiConfiguration,
storageConnectionString,
readAccessGeoRedundant,
requestTimeout);
});
}

private static CloudBlobClientWrapper CreateCloudBlobClient(
StorageMsiConfiguration msiConfiguration,
string storageConnectionString,
bool readAccessGeoRedundant = false,
TimeSpan? requestTimeout = null)
{
if (msiConfiguration.UseManagedIdentity)
{
if (string.IsNullOrWhiteSpace(msiConfiguration.ManagedIdentityClientId))
{
return CloudBlobClientWrapper.UsingDefaultAzureCredential(
storageConnectionString,
readAccessGeoRedundant: readAccessGeoRedundant,
requestTimeout: requestTimeout);
}
else
{
return CloudBlobClientWrapper.UsingMsi(
storageConnectionString,
msiConfiguration.ManagedIdentityClientId,
readAccessGeoRedundant,
requestTimeout);
}
}

return new CloudBlobClientWrapper(
storageConnectionString,
readAccessGeoRedundant,
requestTimeout);
}
}
}
11 changes: 11 additions & 0 deletions src/NuGet.Jobs.Common/StorageMsiConfiguration.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

namespace NuGet.Jobs
{
public class StorageMsiConfiguration
{
public bool UseManagedIdentity { get; set; }
public string ManagedIdentityClientId { get; set; }
}
}
7 changes: 3 additions & 4 deletions src/NuGet.Jobs.GitHubIndexer/Job.cs
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,15 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
services.AddTransient<IRepositoriesCache, DiskRepositoriesCache>();
services.AddTransient<IConfigFileParser, ConfigFileParser>();
services.AddTransient<IRepoFetcher, RepoFetcher>();
services.AddTransient<ICloudBlobClient>(provider => {
var config = provider.GetRequiredService<IOptionsSnapshot<GitHubIndexerConfiguration>>();
return new CloudBlobClientWrapper(config.Value.StorageConnectionString, config.Value.StorageReadAccessGeoRedundant);
});

services.Configure<GitHubIndexerConfiguration>(configurationRoot.GetSection(GitHubIndexerConfigurationSectionName));
}

protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder, IConfigurationRoot configurationRoot)
{
containerBuilder
.RegisterStorageAccount<GitHubIndexerConfiguration>(c => c.StorageConnectionString, c => c.StorageReadAccessGeoRedundant)
.As<ICloudBlobClient>();
}
}
}
19 changes: 5 additions & 14 deletions src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.Rest;
using NuGet.Jobs;
using NuGet.Protocol;
using NuGet.Services.AzureSearch.Auxiliary2AzureSearch;
using NuGet.Services.AzureSearch.AuxiliaryFiles;
Expand Down Expand Up @@ -108,13 +109,7 @@ private static void RegisterIndexServices(ContainerBuilder containerBuilder, str
private static void RegisterAzureSearchStorageServices(ContainerBuilder containerBuilder, string key)
{
containerBuilder
.Register<ICloudBlobClient>(c =>
{
var options = c.Resolve<IOptionsSnapshot<AzureSearchConfiguration>>();
return new CloudBlobClientWrapper(
options.Value.StorageConnectionString,
requestTimeout: DefaultBlobRequestOptions.ServerTimeout);
})
.RegisterStorageAccount<AzureSearchConfiguration>(c => c.StorageConnectionString, requestTimeout: DefaultBlobRequestOptions.ServerTimeout)
.Keyed<ICloudBlobClient>(key);

containerBuilder
Expand Down Expand Up @@ -221,13 +216,9 @@ private static void RegisterAzureSearchStorageServices(ContainerBuilder containe
private static void RegisterAuxiliaryDataStorageServices(ContainerBuilder containerBuilder, string key)
{
containerBuilder
.Register<ICloudBlobClient>(c =>
{
var options = c.Resolve<IOptionsSnapshot<AuxiliaryDataStorageConfiguration>>();
return new CloudBlobClientWrapper(
options.Value.AuxiliaryDataStorageConnectionString,
requestTimeout: DefaultBlobRequestOptions.ServerTimeout);
})
.RegisterStorageAccount<AuxiliaryDataStorageConfiguration>(
c => c.AuxiliaryDataStorageConnectionString,
requestTimeout: DefaultBlobRequestOptions.ServerTimeout)
.Keyed<ICloudBlobClient>(key);

containerBuilder
Expand Down
4 changes: 3 additions & 1 deletion src/NuGet.Services.Configuration/Constants.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

namespace NuGet.Services.Configuration
Expand All @@ -15,5 +15,7 @@ public static class Constants
public static string KeyVaultStoreNameKey = "KeyVault_StoreName";
public static string KeyVaultStoreLocationKey = "KeyVault_StoreLocation";
public static string KeyVaultSendX5c = "KeyVault_SendX5c";
public static string StorageUseManagedIdentityPropertyName = "Storage_UseManagedIdentity";
public static string StorageManagedIdentityClientIdPropertyName = "Storage_ManagedIdentityClientId";
}
}
1 change: 1 addition & 0 deletions src/NuGet.Services.V3/NuGet.Services.V3.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="WindowsAzure.Storage" />
<ProjectReference Include="..\Catalog\NuGet.Services.Metadata.Catalog.csproj" />
<ProjectReference Include="..\Validation.Common.Job\Validation.Common.Job.csproj" />
</ItemGroup>
Expand Down
71 changes: 53 additions & 18 deletions src/NuGet.Services.Validation.Orchestrator/Job.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using System.Threading.Tasks;
using Autofac;
using Autofac.Core;
using Azure.Identity;
using Azure.Storage.Blobs;
using Microsoft.ApplicationInsights;
using Microsoft.Extensions.Configuration;
Expand Down Expand Up @@ -180,13 +181,6 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
services.AddTransient<IBrokeredMessageSerializer<PackageValidationMessageData>, PackageValidationMessageDataSerializationAdapter>();
services.AddTransient<ICriteriaEvaluator<Package>, PackageCriteriaEvaluator>();
services.AddTransient<IProcessSignatureEnqueuer, ProcessSignatureEnqueuer>();
services.AddTransient<ICloudBlobClient>(c =>
{
var configurationAccessor = c.GetRequiredService<IOptionsSnapshot<ValidationConfiguration>>();
return new CloudBlobClientWrapper(
configurationAccessor.Value.ValidationStorageConnectionString,
readAccessGeoRedundant: false);
});
services.AddTransient<ICloudBlobContainerInformationProvider, GalleryCloudBlobContainerInformationProvider>();
services.AddTransient<ICoreFileStorageService, CloudBlobCoreFileStorageService>();
services.AddTransient<IFileDownloader, FileDownloader>();
Expand Down Expand Up @@ -236,6 +230,10 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi

protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder, IConfigurationRoot configurationRoot)
{
containerBuilder
.RegisterStorageAccount<ValidationConfiguration>(c => c.ValidationStorageConnectionString)
.As<ICloudBlobClient>();

containerBuilder
.Register(c =>
{
Expand Down Expand Up @@ -387,11 +385,9 @@ private static void ConfigureLeaseService(ContainerBuilder builder)
.Register(c =>
{
LeaseConfiguration config = c.Resolve<IOptionsSnapshot<LeaseConfiguration>>().Value;
StorageMsiConfiguration storageMsiConfiguration = c.Resolve<IOptionsSnapshot<StorageMsiConfiguration>>().Value;
// workaround for https://github.com/Azure/azure-sdk-for-net/issues/44373
var connectionString = config.ConnectionString.Replace("SharedAccessSignature=?", "SharedAccessSignature=");
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
BlobServiceClient blobServiceClient = CreateBlobServiceClient(storageMsiConfiguration, config.ConnectionString);
return new CloudBlobLeaseService(blobServiceClient, config.ContainerName, config.StoragePath);
})
.As<ILeaseService>();
Expand Down Expand Up @@ -456,13 +452,7 @@ private static void ConfigureSymbolScanValidator(ContainerBuilder builder)
private static void ConfigureFlatContainer(ContainerBuilder builder)
{
builder
.Register<CloudBlobClientWrapper>(c =>
{
var configurationAccessor = c.Resolve<IOptionsSnapshot<FlatContainerConfiguration>>();
return new CloudBlobClientWrapper(
configurationAccessor.Value.ConnectionString,
readAccessGeoRedundant: false);
})
.RegisterStorageAccount<FlatContainerConfiguration>(c => c.ConnectionString)
.Keyed<ICloudBlobClient>(FlatContainerBindingKey);

builder
Expand Down Expand Up @@ -614,5 +604,50 @@ private T GetRequiredService<T>()
{
return _serviceProvider.GetRequiredService<T>();
}

private static BlobServiceClient CreateBlobServiceClient(
StorageMsiConfiguration msiConfiguration,
string storageConnectionString,
TimeSpan? requestTimeout = null)
{
BlobClientOptions blobClientOptions = new BlobClientOptions();
if (requestTimeout.HasValue)
{
blobClientOptions.Retry.NetworkTimeout = requestTimeout.Value;
}

if (msiConfiguration.UseManagedIdentity)
{
if (string.IsNullOrWhiteSpace(msiConfiguration.ManagedIdentityClientId))
{
// 1. Using MSI with DefaultAzureCredential (local debugging)
var defaultAzureCredentialOptions = new DefaultAzureCredentialOptions
{
ManagedIdentityClientId = null,
};

return new BlobServiceClient(
ConnectionStringExtensions.GetBlobEndpointFromConnectionString(storageConnectionString),
new DefaultAzureCredential(defaultAzureCredentialOptions),
blobClientOptions);
}
else
{
// 2. Using MSI with ClientId
return new BlobServiceClient(
ConnectionStringExtensions.GetBlobEndpointFromConnectionString(storageConnectionString),
new ManagedIdentityCredential(msiConfiguration.ManagedIdentityClientId),
blobClientOptions);
}
}
else
{
// 3. Using SAS token
// workaround for https://github.com/Azure/azure-sdk-for-net/issues/44373
var connectionString = storageConnectionString.Replace("SharedAccessSignature=?", "SharedAccessSignature=");

return new BlobServiceClient(connectionString, blobClientOptions);
}
}
}
}
Loading

0 comments on commit 774816d

Please sign in to comment.