@@ -1107,7 +1107,6 @@ @MakeLicenseSpan(segment); } } - @:license } else { diff --git a/src/NuGetGallery/Web.config b/src/NuGetGallery/Web.config index 36d12beba9..dd5d2bdfbd 100644 --- a/src/NuGetGallery/Web.config +++ b/src/NuGetGallery/Web.config @@ -203,6 +203,7 @@ + diff --git a/src/Stats.AzureCdnLogs.Common/Collect/Collector.cs b/src/Stats.AzureCdnLogs.Common/Collect/Collector.cs index e54ecc01fe..b1f81f8d65 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/Collector.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/Collector.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -25,6 +25,8 @@ public abstract class Collector protected ILogSource _source; protected ILogDestination _destination; protected readonly ILogger _logger; + protected readonly bool _writeHeader; + protected readonly bool _addSourceFilenameColumn; /// /// Used by UnitTests @@ -38,11 +40,18 @@ public Collector() /// The source of the Collector. /// The destination for the collector. /// The logger. - public Collector(ILogSource source, ILogDestination destination, ILogger logger) + public Collector( + ILogSource source, + ILogDestination destination, + ILogger logger, + bool writeHeader, + bool addSourceFilenameColumn) { _source = source; _destination = destination; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _writeHeader = writeHeader; + _addSourceFilenameColumn = addSourceFilenameColumn; } /// @@ -126,9 +135,25 @@ private void AddException(ConcurrentBag exceptions, Exception e, stri /// A method to transform each line from the input stream before writing it to the output stream. It is useful for example to modify the schema of each line. /// /// A line from the input stream. + /// The name of the file the was read from. /// The transformed line. public abstract OutputLogLine TransformRawLogLine(string line); + /// + /// A method that returns the file header to put into the output file (if enabled). + /// + /// The header line. + public string GetOutputFileHeader() + { + if (!_addSourceFilenameColumn) + { + return OutputLogLine.Header; + } + + return $"{OutputLogLine.Header} sourceFilename"; + } + + /// /// A method to validate the stream integrity before data transfer. /// @@ -151,7 +176,10 @@ protected void ProcessLogStream(Stream sourceStream, Stream targetStream, string using (var sourceStreamReader = new StreamReader(sourceStream)) using (var targetStreamWriter = new StreamWriter(targetStream)) { - targetStreamWriter.WriteLine(OutputLogLine.Header); + if (_writeHeader) + { + targetStreamWriter.WriteLine(GetOutputFileHeader()); + } while (!sourceStreamReader.EndOfStream) { @@ -279,7 +307,15 @@ private string GetParsedModifiedLogEntry(int lineNumber, string rawLogEntry, str stringBuilder.Append(spaceCharacter); // x-ec_custom-1 - stringBuilder.AppendLine((parsedEntry.CustomField ?? dashCharacter)); + stringBuilder.Append((parsedEntry.CustomField ?? dashCharacter)); + + if (_addSourceFilenameColumn) + { + stringBuilder.Append(spaceCharacter); + stringBuilder.Append(OutputLogLine.Quote(filename)); + } + + stringBuilder.AppendLine(); return stringBuilder.ToString(); } diff --git a/src/Stats.AzureCdnLogs.Common/Collect/OutputLogLine.cs b/src/Stats.AzureCdnLogs.Common/Collect/OutputLogLine.cs index bcc68454ce..6435cccc7a 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/OutputLogLine.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/OutputLogLine.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. namespace Stats.AzureCdnLogs.Common.Collect @@ -79,7 +79,7 @@ public OutputLogLine(string timestamp, public static string Header { - get { return "#Fields: timestamp time-taken c-ip filesize s-ip s-port sc-status sc-bytes cs-method cs-uri-stem - rs-duration rs-bytes c-referrer c-user-agent customer-id x-ec_custom-1\n"; } + get { return "#Fields: timestamp time-taken c-ip filesize s-ip s-port sc-status sc-bytes cs-method cs-uri-stem - rs-duration rs-bytes c-referrer c-user-agent customer-id x-ec_custom-1"; } } public override string ToString() @@ -87,7 +87,7 @@ public override string ToString() return $"{TimeStamp} {TimeTaken} {CIp} {FileSize} {SIp} {SPort} {ScStatus} {ScBytes} {CsMethod} {CsUriStem} - {RsDuration} {RsBytes} {CReferrer} {Quote(CUserAgent)} {CustomerId} {Quote(XEc_Custom_1)}"; } - private static string Quote(string input) + public static string Quote(string input) { if (input.StartsWith("\"") && input.EndsWith("\"")) { diff --git a/src/Stats.CollectAzureChinaCDNLogs/ChinaStatsCollector.cs b/src/Stats.CollectAzureChinaCDNLogs/ChinaStatsCollector.cs index 75333be1d8..efa535c475 100644 --- a/src/Stats.CollectAzureChinaCDNLogs/ChinaStatsCollector.cs +++ b/src/Stats.CollectAzureChinaCDNLogs/ChinaStatsCollector.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -35,7 +35,13 @@ private enum ChinaLogHeaderFields sip = 11 } - public ChinaStatsCollector(ILogSource source, ILogDestination destination, ILogger logger) : base(source, destination, logger) + public ChinaStatsCollector( + ILogSource source, + ILogDestination destination, + ILogger logger, + bool writeHeader, + bool addSourceFilenameColumn) + : base(source, destination, logger, writeHeader, addSourceFilenameColumn) {} public override OutputLogLine TransformRawLogLine(string line) diff --git a/src/Stats.CollectAzureChinaCDNLogs/Configuration/CollectAzureChinaCdnLogsConfiguration.cs b/src/Stats.CollectAzureChinaCDNLogs/Configuration/CollectAzureChinaCdnLogsConfiguration.cs index 2674642093..5d602eaa35 100644 --- a/src/Stats.CollectAzureChinaCDNLogs/Configuration/CollectAzureChinaCdnLogsConfiguration.cs +++ b/src/Stats.CollectAzureChinaCDNLogs/Configuration/CollectAzureChinaCdnLogsConfiguration.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. namespace Stats.CollectAzureChinaCDNLogs @@ -16,5 +16,8 @@ public class CollectAzureChinaCdnLogsConfiguration public string DestinationFilePrefix { get; set; } public int? ExecutionTimeoutInSeconds { get; set; } + + public bool WriteOutputHeader { get; set; } = true; + public bool AddSourceFilenameColumn { get; set; } = false; } } diff --git a/src/Stats.CollectAzureChinaCDNLogs/Job.cs b/src/Stats.CollectAzureChinaCDNLogs/Job.cs index 92d45d626d..78c4c3fef8 100644 --- a/src/Stats.CollectAzureChinaCDNLogs/Job.cs +++ b/src/Stats.CollectAzureChinaCDNLogs/Job.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -53,7 +53,12 @@ public void InitializeJobConfiguration(IServiceProvider serviceProvider) _configuration.AzureContainerNameDestination, serviceProvider.GetRequiredService>()); - _chinaCollector = new ChinaStatsCollector(source, dest, serviceProvider.GetRequiredService>()); + _chinaCollector = new ChinaStatsCollector( + source, + dest, + serviceProvider.GetRequiredService>(), + _configuration.WriteOutputHeader, + _configuration.AddSourceFilenameColumn); } public override async Task Run() diff --git a/src/Stats.PostProcessReports/Job.cs b/src/Stats.PostProcessReports/Job.cs index 0387e3ed88..df6f87a51b 100644 --- a/src/Stats.PostProcessReports/Job.cs +++ b/src/Stats.PostProcessReports/Job.cs @@ -39,7 +39,7 @@ protected override void ConfigureAutofacServices(ContainerBuilder containerBuild .Register(c => { var cfg = c.Resolve>().Value; - return new BlobServiceClient(AzureStorageFactory.PrepareConnectionString(cfg.StorageAccount)); + return new BlobServiceClientFactory(AzureStorageFactory.PrepareConnectionString(cfg.StorageAccount)); }) .AsSelf(); @@ -48,7 +48,7 @@ protected override void ConfigureAutofacServices(ContainerBuilder containerBuild { var cfg = c.Resolve>().Value; var factory = new AzureStorageFactory( - c.Resolve(), + c.Resolve(), cfg.SourceContainerName, enablePublicAccess: false, c.Resolve>(), @@ -65,7 +65,7 @@ protected override void ConfigureAutofacServices(ContainerBuilder containerBuild { var cfg = c.Resolve>().Value; var factory = new AzureStorageFactory( - c.Resolve(), + c.Resolve(), cfg.WorkContainerName, enablePublicAccess: false, c.Resolve>(), @@ -82,7 +82,7 @@ protected override void ConfigureAutofacServices(ContainerBuilder containerBuild { var cfg = c.Resolve>().Value; var factory = new AzureStorageFactory( - c.Resolve(), + c.Resolve(), cfg.DestinationContainerName, enablePublicAccess: true, c.Resolve>(), diff --git a/src/Validation.PackageSigning.ProcessSignature/Job.cs b/src/Validation.PackageSigning.ProcessSignature/Job.cs index 2716a2c754..392023f397 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Job.cs +++ b/src/Validation.PackageSigning.ProcessSignature/Job.cs @@ -55,7 +55,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi var useStorageManagedIdentity = bool.Parse(configurationRoot[Constants.StorageUseManagedIdentityPropertyName]); var config = p.GetRequiredService>().Value; - BlobServiceClient targetStorageAccount; + BlobServiceClientFactory targetStorageAccount; if (useStorageManagedIdentity) { var managedIdentityClientId = @@ -63,12 +63,12 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi configurationRoot[Constants.ManagedIdentityClientIdKey] : configurationRoot[Constants.StorageManagedIdentityClientIdPropertyName]; var storageAccountUri = AzureStorage.GetPrimaryServiceUri(config.DataStorageAccount); - var managedIdentity = new ManagedIdentityCredential(managedIdentityClientId); - targetStorageAccount = new BlobServiceClient(storageAccountUri, managedIdentity); + var managedIdentityCredential = new ManagedIdentityCredential(managedIdentityClientId); + targetStorageAccount = new BlobServiceClientFactory(storageAccountUri, managedIdentityCredential); } else { - targetStorageAccount = new BlobServiceClient(AzureStorageFactory.PrepareConnectionString(config.DataStorageAccount)); + targetStorageAccount = new BlobServiceClientFactory(AzureStorageFactory.PrepareConnectionString(config.DataStorageAccount)); } var storageFactory = new AzureStorageFactory( diff --git a/src/Validation.PackageSigning.ValidateCertificate/Job.cs b/src/Validation.PackageSigning.ValidateCertificate/Job.cs index 883ede5a1f..965faf991f 100644 --- a/src/Validation.PackageSigning.ValidateCertificate/Job.cs +++ b/src/Validation.PackageSigning.ValidateCertificate/Job.cs @@ -37,7 +37,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi var useStorageManagedIdentity = bool.Parse(configurationRoot[Constants.StorageUseManagedIdentityPropertyName]); var config = p.GetRequiredService>().Value; - BlobServiceClient targetStorageAccount; + BlobServiceClientFactory targetStorageAccount; if (useStorageManagedIdentity) { var managedIdentityClientId = @@ -45,12 +45,12 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi configurationRoot[Constants.ManagedIdentityClientIdKey] : configurationRoot[Constants.StorageManagedIdentityClientIdPropertyName]; var storageAccountUri = AzureStorage.GetPrimaryServiceUri(config.DataStorageAccount); - var managedIdentity = new ManagedIdentityCredential(managedIdentityClientId); - targetStorageAccount = new BlobServiceClient(storageAccountUri, managedIdentity); + var managedIdentityCredential = new ManagedIdentityCredential(managedIdentityClientId); + targetStorageAccount = new BlobServiceClientFactory(storageAccountUri, managedIdentityCredential); } else { - targetStorageAccount = new BlobServiceClient(AzureStorageFactory.PrepareConnectionString(config.DataStorageAccount)); + targetStorageAccount = new BlobServiceClientFactory(AzureStorageFactory.PrepareConnectionString(config.DataStorageAccount)); } var storageFactory = new AzureStorageFactory( diff --git a/src/VerifyMicrosoftPackage/Fakes/FakeFeatureFlagService.cs b/src/VerifyMicrosoftPackage/Fakes/FakeFeatureFlagService.cs index 4d89d4fb4a..7b3ab36c67 100644 --- a/src/VerifyMicrosoftPackage/Fakes/FakeFeatureFlagService.cs +++ b/src/VerifyMicrosoftPackage/Fakes/FakeFeatureFlagService.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -135,5 +135,7 @@ public class FakeFeatureFlagService : IFeatureFlagService public bool IsDisplayTfmBadgesEnabled(User user) => throw new NotImplementedException(); public bool IsAdvancedFrameworkFilteringEnabled(User user) => throw new NotImplementedException(); + + public bool CanUseFederatedCredentials(User user) => throw new NotImplementedException(); } } diff --git a/tests/CatalogMetadataTests/AzureStorageFacts.cs b/tests/CatalogMetadataTests/AzureStorageFacts.cs index 349843be1b..9089afed9b 100644 --- a/tests/CatalogMetadataTests/AzureStorageFacts.cs +++ b/tests/CatalogMetadataTests/AzureStorageFacts.cs @@ -10,6 +10,8 @@ using NuGet.Services.Metadata.Catalog.Persistence; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; +using NuGet.Services.Storage; +using AzureStorage = NuGet.Services.Metadata.Catalog.Persistence.AzureStorage; namespace CatalogMetadataTests { @@ -138,14 +140,12 @@ private ICloudBlockBlob GetMockedBlockBlobWithNullMetadata(bool isBlobMetadataEx public abstract class AzureStorageBaseFacts { + protected readonly string _baseAddressString = "DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=fake"; protected readonly Uri _baseAddress = new Uri("https://test"); protected readonly AzureStorage _storage; public AzureStorageBaseFacts() { - // Mock the BlobServiceClient - var mockBlobServiceClient = new Mock(_baseAddress, null); - mockBlobServiceClient.Setup(x => x.Uri).Returns(_baseAddress); // Mock the BlobContainerClient string containerName = "azuresearch"; @@ -159,12 +159,24 @@ public AzureStorageBaseFacts() mockBlobContainerClient.Setup(client => client.Name) .Returns(containerName); + + // Mock the BlobServiceClient + var mockBlobServiceClient = new Mock(_baseAddressString); + mockBlobServiceClient.Setup(x => x.Uri).Returns(_baseAddress); + mockBlobServiceClient.Setup(x => x.GetBlobContainerClient(It.IsAny())) + .Returns(mockBlobContainerClient.Object); + + // Mock the BlobServiceClientFactory + var mockBlobServiceFactoryClient = new Mock(_baseAddressString); + mockBlobServiceFactoryClient.Setup(x => x.Uri).Returns(_baseAddress); + mockBlobServiceFactoryClient.Setup(x => x.GetBlobServiceClient(It.IsAny())) + .Returns(mockBlobServiceClient.Object); // Mock ICloudBlobDirectory var directory = new Mock(); // Setup the ServiceClient to return the mocked BlobServiceClient - directory.Setup(x => x.ServiceClient).Returns(mockBlobServiceClient.Object); + directory.Setup(x => x.ServiceClient).Returns(mockBlobServiceFactoryClient.Object); directory.Setup(x => x.DirectoryPrefix).Returns(""); directory.Setup(x => x.ContainerClientWrapper).Returns(new BlobContainerClientWrapper(mockBlobContainerClient.Object)); diff --git a/tests/CatalogMetadataTests/CloudBlobDirectoryWrapperFacts.cs b/tests/CatalogMetadataTests/CloudBlobDirectoryWrapperFacts.cs index bdc9355310..8510d2f8d9 100644 --- a/tests/CatalogMetadataTests/CloudBlobDirectoryWrapperFacts.cs +++ b/tests/CatalogMetadataTests/CloudBlobDirectoryWrapperFacts.cs @@ -1,10 +1,10 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using Xunit; using NuGet.Services.Metadata.Catalog.Persistence; -using Azure.Storage.Blobs; +using NuGet.Services.Storage; +using Xunit; namespace CatalogMetadataTests { @@ -13,13 +13,13 @@ public class CloudBlobDirectoryWrapperFacts public class TheUriProperty { [Theory] - [InlineData("", "https://test/containerName/")] - [InlineData("directoryPrefix", "https://test/containerName/directoryPrefix")] - [InlineData("directoryPrefix/foo", "https://test/containerName/directoryPrefix/foo")] + [InlineData("", "https://devstoreaccount1.blob.core.windows.net/containerName/")] + [InlineData("directoryPrefix", "https://devstoreaccount1.blob.core.windows.net/containerName/directoryPrefix")] + [InlineData("directoryPrefix/foo", "https://devstoreaccount1.blob.core.windows.net/containerName/directoryPrefix/foo")] public void ReturnsTheUriWithVariousPrefixes(string directoryPrefix, string expectedUri) { // Arrange - var serviceClient = new BlobServiceClient(new Uri("https://test")); + var serviceClient = new BlobServiceClientFactory("DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=fake"); var directory = new CloudBlobDirectoryWrapper(serviceClient, "containerName", directoryPrefix); // Act diff --git a/tests/NgTests/CommandHelpersTests.cs b/tests/NgTests/CommandHelpersTests.cs index 370768351b..7afd1e247d 100644 --- a/tests/NgTests/CommandHelpersTests.cs +++ b/tests/NgTests/CommandHelpersTests.cs @@ -202,6 +202,7 @@ public void DefaultsToAzurePublicWithNoCompression() { Arguments.StorageKeyValue, DummyKey }, { Arguments.StorageContainer, "testContainer" }, { Arguments.StoragePath, "testStoragePath" }, + { Arguments.StorageUseManagedIdentity, "false" }, }; // Act @@ -226,6 +227,7 @@ public void AllowsCustomStorageSuffix() { Arguments.StorageContainer, "testContainer" }, { Arguments.StoragePath, "testStoragePath" }, { Arguments.StorageSuffix, "core.chinacloudapi.cn" }, + { Arguments.StorageUseManagedIdentity, "false" } }; // Act diff --git a/tests/NuGet.Services.Storage.Tests/AzureStorageNewFacts.cs b/tests/NuGet.Services.Storage.Tests/AzureStorageNewFacts.cs index cd05baea83..3f5141d029 100644 --- a/tests/NuGet.Services.Storage.Tests/AzureStorageNewFacts.cs +++ b/tests/NuGet.Services.Storage.Tests/AzureStorageNewFacts.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -21,6 +21,7 @@ public class AzureStorageNewFacts { private Mock _blobClientMock = new Mock(); private Mock _blobServiceClientMock = new Mock(); + private Mock _blobServiceClientFactoryMock = new Mock(); private Mock _blobContainerClientMock = new Mock(); private Mock> _loggerMock = new Mock>(); @@ -34,13 +35,16 @@ public AzureStorageNewFacts() _blobServiceClientMock .Setup(x => x.GetBlobContainerClient(It.IsAny())) .Returns(_blobContainerClientMock.Object); + _blobServiceClientFactoryMock + .Setup(x => x.GetBlobServiceClient(It.IsAny())) + .Returns(_blobServiceClientMock.Object); } [Fact] public void Constructor_DoNotInitialize() { var azureStorage = new AzureStorage( - _blobServiceClientMock.Object, + _blobServiceClientFactoryMock.Object, "containerName", "path", new Uri("http://baseAddress"), @@ -75,7 +79,7 @@ public void Constructor_Initialize_ContainerDoesNotExist(bool enablePublicAccess .Throws(new RequestFailedException(404, "Not found")); var azureStorage = new AzureStorage( - _blobServiceClientMock.Object, + _blobServiceClientFactoryMock.Object, "containerName", "path", new Uri("http://baseAddress"), @@ -109,7 +113,7 @@ public void Constructor_Initialize_ContainerExists_AccessMatches(bool enablePubl Mock.Of())); var azureStorage = new AzureStorage( - _blobServiceClientMock.Object, + _blobServiceClientFactoryMock.Object, "containerName", "path", new Uri("http://baseAddress"), @@ -143,7 +147,7 @@ public void Constructor_Initialize_ContainerExists_AccessDoesNotMatch(bool enabl Mock.Of())); var azureStorage = new AzureStorage( - _blobServiceClientMock.Object, + _blobServiceClientFactoryMock.Object, "containerName", "path", new Uri("http://baseAddress"), @@ -172,7 +176,7 @@ public void Exists(bool expected) _blobClientMock.Setup(c => c.Exists(It.IsAny())).Returns(Response.FromValue(expected, azureResponse.Object)); _blobContainerClientMock.Protected().Setup("GetBlockBlobClientCore",ItExpr.IsAny()) .Returns(_blobClientMock.Object); - var azureStorage = new AzureStorage(_blobServiceClientMock.Object, "containerName", "path", new Uri("http://baseAddress"), initializeContainer: true, enablePublicAccess: false, _loggerMock.Object); + var azureStorage = new AzureStorage(_blobServiceClientFactoryMock.Object, "containerName", "path", new Uri("http://baseAddress"), initializeContainer: true, enablePublicAccess: false, _loggerMock.Object); Assert.Equal(azureStorage.Exists("file"), expected); } @@ -186,7 +190,7 @@ public async Task ExistsAsync(bool expected) _blobClientMock.Setup(c => c.ExistsAsync(It.IsAny())).Returns(Task.FromResult(Response.FromValue(expected, azureResponse.Object))); _blobContainerClientMock.Protected().Setup("GetBlockBlobClientCore", ItExpr.IsAny()) .Returns(_blobClientMock.Object); - var azureStorage = new AzureStorage(_blobServiceClientMock.Object, "containerName", "path", new Uri("http://baseAddress"), initializeContainer: true, enablePublicAccess: false, _loggerMock.Object); + var azureStorage = new AzureStorage(_blobServiceClientFactoryMock.Object, "containerName", "path", new Uri("http://baseAddress"), initializeContainer: true, enablePublicAccess: false, _loggerMock.Object); Assert.Equal(await azureStorage.ExistsAsync("file", CancellationToken.None), expected); } @@ -211,7 +215,7 @@ public async Task Save(bool overwrite, bool exists, int calledTimes) _blobContainerClientMock.Protected().Setup("GetBlockBlobClientCore", ItExpr.IsAny()) .Returns(_blobClientMock.Object); - var azureStorage = new AzureStorage(_blobServiceClientMock.Object, "containerName", "path", new Uri("http://baseAddress"), initializeContainer: true, enablePublicAccess: false, _loggerMock.Object); + var azureStorage = new AzureStorage(_blobServiceClientFactoryMock.Object, "containerName", "path", new Uri("http://baseAddress"), initializeContainer: true, enablePublicAccess: false, _loggerMock.Object); await azureStorage.Save(new Uri("http://testUri.com/blob.json"), new StringStorageContent("content"), overwrite: overwrite, CancellationToken.None); diff --git a/tests/NuGetGallery.Core.Facts/Auditing/CredentialAuditRecordTests.cs b/tests/NuGetGallery.Core.Facts/Auditing/CredentialAuditRecordTests.cs index 3d21a1cbd9..e1828b7e03 100644 --- a/tests/NuGetGallery.Core.Facts/Auditing/CredentialAuditRecordTests.cs +++ b/tests/NuGetGallery.Core.Facts/Auditing/CredentialAuditRecordTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -13,31 +13,23 @@ public class CredentialAuditRecordTests [Fact] public void Constructor_ThrowsForNullCredential() { - Assert.Throws(() => new CredentialAuditRecord(credential: null, removedOrRevoked: true)); + Assert.Throws(() => new CredentialAuditRecord(credential: null)); } [Fact] - public void Constructor_ThrowsForRemovalWithNullType() - { - var credential = new Credential(); - - Assert.Throws(() => new CredentialAuditRecord(credential, removedOrRevoked: true)); - } - - [Fact] - public void Constructor_RemovalOfNonPasswordSetsValue() + public void Constructor_RemovalOfNonPasswordDoesNotSetValue() { var credential = new Credential(type: "a", value: "b"); - var record = new CredentialAuditRecord(credential, removedOrRevoked: true); + var record = new CredentialAuditRecord(credential); - Assert.Equal("b", record.Value); + Assert.Null(record.Value); } [Fact] public void Constructor_RemovalOfPasswordDoesNotSetValue() { var credential = new Credential(type: CredentialTypes.Password.V3, value: "a"); - var record = new CredentialAuditRecord(credential, removedOrRevoked: true); + var record = new CredentialAuditRecord(credential); Assert.Null(record.Value); } @@ -46,7 +38,7 @@ public void Constructor_RemovalOfPasswordDoesNotSetValue() public void Constructor_NonRemovalOfNonPasswordDoesNotSetsValue() { var credential = new Credential(type: "a", value: "b"); - var record = new CredentialAuditRecord(credential, removedOrRevoked: false); + var record = new CredentialAuditRecord(credential); Assert.Null(record.Value); } @@ -57,7 +49,7 @@ public void Constructor_NonRemovalOfNonPasswordDoesNotSetsValue() public void Constructor_ExternalCredentialSetsValue(string externalType) { var credential = new Credential(type: externalType, value: "b"); - var record = new CredentialAuditRecord(credential, removedOrRevoked: false); + var record = new CredentialAuditRecord(credential); Assert.Equal("b", record.Value); } @@ -66,7 +58,7 @@ public void Constructor_ExternalCredentialSetsValue(string externalType) public void Constructor_NonRemovalOfPasswordDoesNotSetValue() { var credential = new Credential(type: CredentialTypes.Password.V3, value: "a"); - var record = new CredentialAuditRecord(credential, removedOrRevoked: false); + var record = new CredentialAuditRecord(credential); Assert.Null(record.Value); } @@ -90,7 +82,7 @@ public void Constructor_SetsProperties() Type = "e", Value = "f" }; - var record = new CredentialAuditRecord(credential, removedOrRevoked: true); + var record = new CredentialAuditRecord(credential); Assert.Equal(created, record.Created); Assert.Equal("a", record.Description); @@ -104,7 +96,7 @@ public void Constructor_SetsProperties() Assert.Equal("c", scope.Subject); Assert.Equal("d", scope.AllowedAction); Assert.Equal("e", record.Type); - Assert.Equal("f", record.Value); + Assert.Null(record.Value); } [Fact] @@ -112,11 +104,11 @@ public void Constructor_WithRevocationSource_Properties() { var testRevocationSource = "TestRevocationSource"; var credential = new Credential(type: "a", value: "b"); - var record = new CredentialAuditRecord(credential, removedOrRevoked: true, revocationSource: testRevocationSource); + var record = new CredentialAuditRecord(credential, revocationSource: testRevocationSource); Assert.Equal(testRevocationSource, record.RevocationSource); Assert.Equal("a", record.Type); - Assert.Equal("b", record.Value); + Assert.Null(record.Value); } } -} \ No newline at end of file +} diff --git a/tests/NuGetGallery.Core.Facts/Auditing/UserAuditRecordTests.cs b/tests/NuGetGallery.Core.Facts/Auditing/UserAuditRecordTests.cs index 6760589e61..ca42318626 100644 --- a/tests/NuGetGallery.Core.Facts/Auditing/UserAuditRecordTests.cs +++ b/tests/NuGetGallery.Core.Facts/Auditing/UserAuditRecordTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -108,7 +108,7 @@ public void Constructor_WithRevocationSource_SetsProperties() Assert.Single(record.AffectedCredential); Assert.Equal(testRevocationSource, record.AffectedCredential[0].RevocationSource); Assert.Equal("b", record.AffectedCredential[0].Type); - Assert.Equal("c", record.AffectedCredential[0].Value); + Assert.Null(record.AffectedCredential[0].Value); } [Fact] @@ -127,4 +127,4 @@ public void GetPath_ReturnsLowerCasedUserName() Assert.Equal("a", actualPath); } } -} \ No newline at end of file +} diff --git a/tests/NuGetGallery.Core.Facts/Frameworks/FrameworkCompatibilityServiceFacts.cs b/tests/NuGetGallery.Core.Facts/Frameworks/FrameworkCompatibilityServiceFacts.cs index d279bd8cef..cdd0af9eab 100644 --- a/tests/NuGetGallery.Core.Facts/Frameworks/FrameworkCompatibilityServiceFacts.cs +++ b/tests/NuGetGallery.Core.Facts/Frameworks/FrameworkCompatibilityServiceFacts.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -155,9 +155,7 @@ public void PlatformVersionsShouldContainAllSpecifiedFrameworks(string platformV var compatibleFrameworks = FrameworkCompatibilityService.GetCompatibleFrameworks(new HashSet() { packageFramework }); // Assert - Assert.Equal(expectedFrameworks.Length, compatibleFrameworks.Count); - - var containsAllCompatibleFrameworks = compatibleFrameworks.All(cf => projectFrameworks.Contains(cf)); + var containsAllCompatibleFrameworks = projectFrameworks.All(cf => compatibleFrameworks.Contains(cf)); Assert.True(containsAllCompatibleFrameworks); } @@ -189,9 +187,7 @@ public void MultiplePlatformVersionCasesShouldContainAllSpecifiedFrameworks(stri var compatibleFrameworks = FrameworkCompatibilityService.GetCompatibleFrameworks(packageFrameworks); // Assert - Assert.Equal(expectedFrameworks.Length, compatibleFrameworks.Count); - - var containsAllCompatibleFrameworks = compatibleFrameworks.All(cf => expectedComputedFrameworks.Contains(cf)); + var containsAllCompatibleFrameworks = expectedComputedFrameworks.All(cf => compatibleFrameworks.Contains(cf)); Assert.True(containsAllCompatibleFrameworks); } } diff --git a/tests/NuGetGallery.Facts/Authentication/AuthenticationServiceFacts.cs b/tests/NuGetGallery.Facts/Authentication/AuthenticationServiceFacts.cs index ee1a26eb7e..01466c530a 100644 --- a/tests/NuGetGallery.Facts/Authentication/AuthenticationServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Authentication/AuthenticationServiceFacts.cs @@ -1,8 +1,9 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; using System.Collections.Generic; +using System.Data.Entity; using System.Globalization; using System.Linq; using System.Security.Claims; @@ -1605,7 +1606,7 @@ public async Task WritesAuditRecordRemovingTheOldCredential() ar.AffectedCredential.Length == 1 && ar.AffectedCredential[0].Type == existingCred.Type && ar.AffectedCredential[0].Identity == existingCred.Identity && - ar.AffectedCredential[0].Value == existingCred.Value && + ar.AffectedCredential[0].Value is null && ar.AffectedCredential[0].Created == existingCred.Created && ar.AffectedCredential[0].Expires == existingCred.Expires)); } @@ -2382,6 +2383,44 @@ public async Task SavesChangesInTheDataStore() authService.Entities.VerifyCommitChanges(); } + /// + /// Needed to avoid collection modified exception caused by the entity context. + /// + [Fact] + public async Task CopiesScopeCollectionForDeletion() + { + // Arrange + var credentialBuilder = new CredentialBuilder(); + + var credScopes = + Enumerable.Range(0, 5) + .Select( + i => new Scope { AllowedAction = NuGetScopes.PackagePush, Key = i, Subject = "package" + i }).ToList(); + + var mockScopes = new Mock>(); + var dbContext = GetMock(); + dbContext.Setup(x => x.Scopes).Returns(mockScopes.Object); + mockScopes.Setup(x => x.Remove(It.IsAny())).Callback(x => credScopes.Remove(x)); + + var fakes = Get(); + var cred = credentialBuilder.CreateApiKey(null, out string plaintextApiKey); + var user = fakes.CreateUser("test", credentialBuilder.CreatePasswordCredential(Fakes.Password), cred); + var authService = Get(); + + var newScopes = + Enumerable.Range(1, 2) + .Select( + i => new Scope { AllowedAction = NuGetScopes.PackageUnlist, Key = i * 10, Subject = "otherpackage" + i }).ToList(); + + cred.Scopes = credScopes; + + // Act + await authService.EditCredentialScopes(user, cred, newScopes); + + // Act + Assert.Empty(credScopes); + } + [Fact] public async Task WritesAuditRecordForTheEditedCredential() { @@ -2633,4 +2672,4 @@ public static bool VerifyPasswordHash(string hash, string algorithm, string pass return canAuthenticate && !confidenceCheck; } } -} \ No newline at end of file +} diff --git a/tests/NuGetGallery.Facts/Authentication/CredentialBuilderFacts.cs b/tests/NuGetGallery.Facts/Authentication/CredentialBuilderFacts.cs new file mode 100644 index 0000000000..24f273a6cb --- /dev/null +++ b/tests/NuGetGallery.Facts/Authentication/CredentialBuilderFacts.cs @@ -0,0 +1,83 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Services.Entities; +using NuGetGallery.Authentication; +using Xunit; + +namespace NuGetGallery.Infrastructure.Authentication +{ + public class CredentialBuilderFacts + { + public class TheCreateShortLivedApiKeyMethod : CredentialBuilderFacts + { + [Fact] + public void CreatesShortLivedApiKey() + { + // Act + var credential = Target.CreateShortLivedApiKey(Expiration, Policy, out var plaintextApiKey); + + // Assert + Assert.Null(credential.User); + Assert.Equal(default, credential.UserKey); + Assert.StartsWith("oy2", plaintextApiKey, StringComparison.Ordinal); + Assert.Equal(CredentialTypes.ApiKey.V4, credential.Type); + Assert.Equal("Short-lived API key generated via a federated credential", credential.Description); + Assert.Equal(Expiration.Ticks, credential.ExpirationTicks); + Assert.Null(credential.User); + + var scope = Assert.Single(credential.Scopes); + Assert.Equal(NuGetScopes.All, scope.AllowedAction); + Assert.Equal(NuGetPackagePattern.AllInclusivePattern, scope.Subject); + Assert.Same(Policy.PackageOwner, scope.Owner); + } + + [Fact] + public void RejectsMissingPackageOwner() + { + // Arrange + Policy.PackageOwner = null; + + // Act + Assert.Throws(() => Target.CreateShortLivedApiKey(Expiration, Policy, out var plaintextApiKey)); + } + + [Theory] + [InlineData(-1)] + [InlineData(0)] + [InlineData(61)] + public void RejectsOutOfRangeExpiration(int expirationMinutes) + { + // Arrange + Expiration = TimeSpan.FromMinutes(expirationMinutes); + + // Act + Assert.Throws(() => Target.CreateShortLivedApiKey(Expiration, Policy, out var plaintextApiKey)); + } + + public FederatedCredentialPolicy Policy { get; } + + public TheCreateShortLivedApiKeyMethod() + { + Policy = new FederatedCredentialPolicy + { + Key = 23, + PackageOwner = new User { Key = 42 }, + CreatedBy = new User { Key = 43 }, + }; + } + } + + public TimeSpan Expiration { get; set; } + + public CredentialBuilder Target { get; } + + public CredentialBuilderFacts() + { + Expiration = TimeSpan.FromMinutes(13); + + Target = new CredentialBuilder(); + } + } +} diff --git a/tests/NuGetGallery.Facts/Authentication/Federated/EntraIdTokenValidatorFacts.cs b/tests/NuGetGallery.Facts/Authentication/Federated/EntraIdTokenValidatorFacts.cs new file mode 100644 index 0000000000..71797bc3de --- /dev/null +++ b/tests/NuGetGallery.Facts/Authentication/Federated/EntraIdTokenValidatorFacts.cs @@ -0,0 +1,177 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.IdentityModel.JsonWebTokens; +using Microsoft.IdentityModel.Protocols; +using Microsoft.IdentityModel.Protocols.OpenIdConnect; +using Microsoft.IdentityModel.Tokens; +using Moq; +using Xunit; + +#nullable enable + +namespace NuGetGallery.Services.Authentication +{ + public class EntraIdTokenValidatorFacts + { + public class TheValidateAsyncMethod : EntraIdTokenValidatorFacts + { + [Fact] + public async Task RejectsMissingAudience() + { + // Arrange + Configuration.Setup(x => x.EntraIdAudience).Returns((string?)null); + + // Act & Assert + await Assert.ThrowsAsync(() => Target.ValidateAsync(Token)); + } + + [Fact] + public async Task ReturnsTokenValidationResultDirectly() + { + // Arrange + var result = new TokenValidationResult(); + JsonWebTokenHandler + .Setup(x => x.ValidateTokenAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(result); + + // Act + var actual = await Target.ValidateAsync(Token); + + // Assert + Assert.Same(result, actual); + } + + [Fact] + public async Task ConfiguresTokenValidationParameters() + { + // Arrange + var token = Token; + + // Act + await Target.ValidateAsync(token); + + // Assert + JsonWebTokenHandler.Verify(x => x.ValidateTokenAsync(token, It.IsAny()), Times.Once); + var invocation = Assert.Single(JsonWebTokenHandler.Invocations); + var tokenValidationParameters = (TokenValidationParameters)invocation.Arguments[1]; + Assert.Equal("nuget-audience", tokenValidationParameters.ValidAudience); + Assert.NotNull(tokenValidationParameters.IssuerValidator); + Assert.NotNull(tokenValidationParameters.IssuerSigningKeyValidatorUsingConfiguration); + Assert.Same(OidcConfigManager.Object, tokenValidationParameters.ConfigurationManager); + } + + [Fact] + public async Task AcceptsEntraIdIssuer() + { + // Arrange + await Target.ValidateAsync(Token); + JsonWebTokenHandler.Verify(x => x.ValidateTokenAsync(It.IsAny(), It.IsAny()), Times.Once); + var invocation = Assert.Single(JsonWebTokenHandler.Invocations); + var tokenValidationParameters = (TokenValidationParameters)invocation.Arguments[1]; + + // Act + var issuer = tokenValidationParameters.IssuerValidator(Issuer, Token, tokenValidationParameters); + + // Assert + Assert.Equal(Issuer, issuer); + } + + [Fact] + public async Task RejectsOtherIssuer() + { + // Arrange + Issuer = "https://localhost/my-issuer"; + await Target.ValidateAsync(Token); + JsonWebTokenHandler.Verify(x => x.ValidateTokenAsync(It.IsAny(), It.IsAny()), Times.Once); + var invocation = Assert.Single(JsonWebTokenHandler.Invocations); + var tokenValidationParameters = (TokenValidationParameters)invocation.Arguments[1]; + + // Act & Assert + Assert.Throws( + () => tokenValidationParameters.IssuerValidator(Issuer, Token, tokenValidationParameters)); + } + + [Fact] + public async Task AcceptsSigningKeyWithMatchingIssuer() + { + // Arrange + await Target.ValidateAsync(Token); + JsonWebTokenHandler.Verify(x => x.ValidateTokenAsync(It.IsAny(), It.IsAny()), Times.Once); + var invocation = Assert.Single(JsonWebTokenHandler.Invocations); + var tokenValidationParameters = (TokenValidationParameters)invocation.Arguments[1]; + + // Act + var valid = tokenValidationParameters.IssuerSigningKeyValidatorUsingConfiguration(JsonWebKey, Token, tokenValidationParameters, OpenIdConnectConfiguration); + + // Assert + Assert.True(valid); + } + + [Fact] + public async Task RejectsSigningKeyWithDifferentIssuer() + { + // Arrange + await Target.ValidateAsync(Token); + JsonWebTokenHandler.Verify(x => x.ValidateTokenAsync(It.IsAny(), It.IsAny()), Times.Once); + var invocation = Assert.Single(JsonWebTokenHandler.Invocations); + var tokenValidationParameters = (TokenValidationParameters)invocation.Arguments[1]; + var key = new JsonWebKey { Kid = "key-id", AdditionalData = { { "issuer", "https://localhost/my-issuer" } } }; + var config = new OpenIdConnectConfiguration { JsonWebKeySet = new JsonWebKeySet { Keys = { key } } }; + + // Act & Assert + Assert.Throws( + () => tokenValidationParameters.IssuerSigningKeyValidatorUsingConfiguration(key, Token, tokenValidationParameters, config)); + } + } + + public EntraIdTokenValidatorFacts() + { + ConfigurationRetriever = new Mock>(); + OidcConfigManager = new Mock>( + EntraIdTokenValidator.MetadataAddress, + ConfigurationRetriever.Object); + JsonWebTokenHandler = new Mock(); + Configuration = new Mock(); + Configuration.Setup(x => x.EntraIdAudience).Returns("nuget-audience"); + + TenantId = "c311b905-19a2-483e-a014-41d0fcdc99cf"; + Issuer = $"https://login.microsoftonline.com/{TenantId}/v2.0"; + + Target = new EntraIdTokenValidator( + OidcConfigManager.Object, + JsonWebTokenHandler.Object, + Configuration.Object); + } + + public EntraIdTokenValidator Target { get; } + public Mock> ConfigurationRetriever { get; } + public Mock> OidcConfigManager { get; } + public Mock JsonWebTokenHandler { get; } + public Mock Configuration { get; } + public string TenantId { get; } + public string Issuer { get; set; } + + public JsonWebToken Token + { + get + { + var handler = new JsonWebTokenHandler(); + return handler.ReadJsonWebToken(handler.CreateToken(new SecurityTokenDescriptor + { + Claims = new Dictionary + { + { "tid", TenantId }, + { "iss", Issuer }, + } + })); + } + } + + public JsonWebKey JsonWebKey => new() { Kid = "key-id", AdditionalData = { { "issuer", Issuer } } }; + public OpenIdConnectConfiguration OpenIdConnectConfiguration => new() { JsonWebKeySet = new JsonWebKeySet { Keys = { JsonWebKey } } }; + } +} diff --git a/tests/NuGetGallery.Facts/Authentication/Federated/FederatedCredentialRepositoryFacts.cs b/tests/NuGetGallery.Facts/Authentication/Federated/FederatedCredentialRepositoryFacts.cs new file mode 100644 index 0000000000..426c44d219 --- /dev/null +++ b/tests/NuGetGallery.Facts/Authentication/Federated/FederatedCredentialRepositoryFacts.cs @@ -0,0 +1,159 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Moq; +using NuGet.Services.Entities; +using Xunit; + +#nullable enable + +namespace NuGetGallery.Services.Authentication +{ + public class FederatedCredentialRepositoryFacts + { + public class TheGetPoliciesCreatedByUserMethod : FederatedCredentialRepositoryFacts + { + [Fact] + public void FiltersByUserKey() + { + // Act + var policies = Target.GetPoliciesCreatedByUser(userKey: 4); + + // Assert + Assert.Equal(2, policies.Count); + Assert.Equal(1, policies[0].Key); + Assert.Equal(2, policies[1].Key); + } + } + + public class TheGetPolicyByKeyMethod : FederatedCredentialRepositoryFacts + { + [Fact] + public void ReturnsPolicyByKey() + { + // Act + var policy = Target.GetPolicyByKey(2); + + // Assert + Assert.Equal(2, policy!.Key); + } + + [Fact] + public void ReturnsNullIfDoesNotExist() + { + // Act + var policy = Target.GetPolicyByKey(23); + + // Assert + Assert.Null(policy); + } + } + + public class TheSaveFederatedCredentialAsyncMethod : FederatedCredentialRepositoryFacts + { + [Fact] + public async Task InsertsCredential() + { + // Arrange + var credential = new FederatedCredential(); + + // Act + await Target.SaveFederatedCredentialAsync(credential, saveChanges: false); + + // Assert + CredentialRepository.Verify(x => x.InsertOnCommit(credential), Times.Once); + CredentialRepository.Verify(x => x.CommitChangesAsync(), Times.Never); + } + + [Fact] + public async Task CommitsChangesIfRequested() + { + // Arrange + var credential = new FederatedCredential(); + + // Act + await Target.SaveFederatedCredentialAsync(credential, saveChanges: true); + + // Assert + CredentialRepository.Verify(x => x.InsertOnCommit(credential), Times.Once); + CredentialRepository.Verify(x => x.CommitChangesAsync(), Times.Once); + } + } + + public class TheAddPolicyAsyncMethod : FederatedCredentialRepositoryFacts + { + [Fact] + public async Task InsertsPolicy() + { + // Act + await Target.AddPolicyAsync(Policies[0], saveChanges: false); + + // Assert + PolicyRepository.Verify(x => x.InsertOnCommit(Policies[0]), Times.Once); + PolicyRepository.Verify(x => x.CommitChangesAsync(), Times.Never); + } + + [Fact] + public async Task CommitsChangesIfRequested() + { + // Act + await Target.AddPolicyAsync(Policies[0], saveChanges: true); + + // Assert + PolicyRepository.Verify(x => x.InsertOnCommit(Policies[0]), Times.Once); + PolicyRepository.Verify(x => x.CommitChangesAsync(), Times.Once); + } + } + + public class TheDeletePolicyAsyncMethod : FederatedCredentialRepositoryFacts + { + [Fact] + public async Task DeletesPolicy() + { + // Act + await Target.DeletePolicyAsync(Policies[0], saveChanges: false); + + // Assert + PolicyRepository.Verify(x => x.DeleteOnCommit(Policies[0]), Times.Once); + PolicyRepository.Verify(x => x.CommitChangesAsync(), Times.Never); + } + + [Fact] + public async Task CommitsChangesIfRequested() + { + // Act + await Target.DeletePolicyAsync(Policies[0], saveChanges: true); + + // Assert + PolicyRepository.Verify(x => x.DeleteOnCommit(Policies[0]), Times.Once); + PolicyRepository.Verify(x => x.CommitChangesAsync(), Times.Once); + } + } + + public FederatedCredentialRepositoryFacts() + { + CredentialRepository = new Mock>(); + PolicyRepository = new Mock>(); + + Policies = new List + { + new() { Key = 1, CreatedByUserKey = 4 }, + new() { Key = 2, CreatedByUserKey = 4 }, + new() { Key = 3, CreatedByUserKey = 5 }, + }; + PolicyRepository.Setup(x => x.GetAll()).Returns(() => Policies.AsQueryable()); + + Target = new FederatedCredentialRepository( + PolicyRepository.Object, + CredentialRepository.Object); + } + + public Mock> CredentialRepository { get; } + public Mock> PolicyRepository { get; } + public List Policies { get; } + public FederatedCredentialRepository Target { get; } + } +} diff --git a/tests/NuGetGallery.Facts/Services/PackageDeprecationManagementServiceFacts.cs b/tests/NuGetGallery.Facts/Services/PackageDeprecationManagementServiceFacts.cs index 20b424c8f7..4170717547 100644 --- a/tests/NuGetGallery.Facts/Services/PackageDeprecationManagementServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/PackageDeprecationManagementServiceFacts.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -856,7 +856,7 @@ private async Task AssertSuccessful( var package2 = new Package { - NormalizedVersion = "1.0.0", + NormalizedVersion = "1.0.0-RC1", PackageRegistration = registration }; @@ -932,7 +932,7 @@ private async Task AssertSuccessful( var service = GetService(); - var packageNormalizedVersions = new[] { package.NormalizedVersion, package2.NormalizedVersion }; + var packageNormalizedVersions = new[] { package.NormalizedVersion, package2.NormalizedVersion.ToLowerInvariant() }; // Act var result = await InvokeUpdateDeprecation( @@ -981,4 +981,4 @@ private static Task InvokeUpdateDeprecation( } } } -} \ No newline at end of file +} diff --git a/tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs b/tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs index af0a6ae6ff..afe299e6e3 100644 --- a/tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs +++ b/tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -35,7 +35,9 @@ public void TransformRawLogLine(string input, string expectedOutput) var collector = new ChinaStatsCollector( Mock.Of(), Mock.Of(), - Mock.Of>()); + Mock.Of>(), + writeHeader: true, + addSourceFilenameColumn: false); var transformedInput = collector.TransformRawLogLine(input); string output = transformedInput == null ? null : transformedInput.ToString(); @@ -51,7 +53,9 @@ public void CdnLogEntryParserIntegration(string input) var collector = new ChinaStatsCollector( Mock.Of(), Mock.Of(), - Mock.Of>()); + Mock.Of>(), + writeHeader: true, + addSourceFilenameColumn: false); var transformedInput = collector.TransformRawLogLine(input); if (transformedInput == null) @@ -74,7 +78,9 @@ public void SkipsMalformedTimestamps() var collector = new ChinaStatsCollector( Mock.Of(), Mock.Of(), - Mock.Of>()); + Mock.Of>(), + writeHeader: true, + addSourceFilenameColumn: false); OutputLogLine transformedInput = null; @@ -121,24 +127,106 @@ public void SkipsMalformedTimestamps() public async Task SkipsLinesWithMalformedColumns(string data, int expectedOutputLines) { const string header = "c-ip, timestamp, cs-method, cs-uri-stem, http-ver, sc-status, sc-bytes, c-referer, c-user-agent, rs-duration(ms), hit-miss, s-ip\n"; - var sourceUri = new Uri("https://example.com/log1"); + Mock sourceMock = SetupSource(header + data); - var sourceMock = new Mock(); - sourceMock - .Setup(s => s.GetFilesAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((IEnumerable)new List { sourceUri }); + var writeSucceeded = false; + var outputStream = new MemoryStream(); + Mock destinationMock = SetupDestination(outputStream, () => writeSucceeded = true); - sourceMock - .Setup(s => s.TakeLockAsync(sourceUri, It.IsAny())) - .ReturnsAsync(new AzureBlobLockResult(new Microsoft.WindowsAzure.Storage.Blob.CloudBlob(sourceUri), true, "foo", CancellationToken.None)); + var collector = new ChinaStatsCollector( + sourceMock.Object, + destinationMock.Object, + Mock.Of>(), + writeHeader: true, + addSourceFilenameColumn: false); - sourceMock - .Setup(s => s.OpenReadAsync(sourceUri, It.IsAny(), It.IsAny())) - .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(header + data))); + await collector.TryProcessAsync( + maxFileCount: 10, + fileNameTransform: s => s, + sourceContentType: ContentType.Text, + destinationContentType: ContentType.Text, + CancellationToken.None); - var destinationMock = new Mock(); + string[] outputLines = null; + + outputLines = GetStreamLines(outputStream); + + Assert.True(writeSucceeded); + Assert.NotEmpty(outputLines); + Assert.Equal("#Fields: timestamp time-taken c-ip filesize s-ip s-port sc-status sc-bytes cs-method cs-uri-stem - rs-duration rs-bytes c-referrer c-user-agent customer-id x-ec_custom-1", outputLines[0]); + Assert.True(expectedOutputLines <= outputLines.Length - 1); // excluding header + } + + [Fact] + public async Task DoesNotWriteHeaderIfConfigured() + { + const string header = "c-ip, timestamp, cs-method, cs-uri-stem, http-ver, sc-status, sc-bytes, c-referer, c-user-agent, rs-duration(ms), hit-miss, s-ip\n"; + const string data = "1.2.3.4,4/6/2019 4:00:20 PM +00:00,GET,\"/v3-flatcontainer/microsoft.codeanalysis.common/1.2.2/microsoft.codeanalysis.common.1.2.2.nupkg\",HTTPS,200,2044843,\"NULL\",\"NuGet VS VSIX/4.7.0 (Microsoft Windows NT 10.0.17134.0, VS Enterprise/15.0)\",796,MISS,4.3.2.1"; + + Mock sourceMock = SetupSource(header + data); + var writeSucceeded = false; var outputStream = new MemoryStream(); + Mock destinationMock = SetupDestination(outputStream, () => writeSucceeded = true); + + var collector = new ChinaStatsCollector( + sourceMock.Object, + destinationMock.Object, + Mock.Of>(), + writeHeader: false, + addSourceFilenameColumn: false); + + await collector.TryProcessAsync( + maxFileCount: 10, + fileNameTransform: s => s, + sourceContentType: ContentType.Text, + destinationContentType: ContentType.Text, + CancellationToken.None); + + string[] outputLines = null; + + outputLines = GetStreamLines(outputStream); + Assert.True(writeSucceeded); + Assert.Single(outputLines); + Assert.False(outputLines[0].StartsWith("#Fields")); + } + + [Fact] + public async Task WritesSourceFilenameColumn() + { + const string header = "c-ip, timestamp, cs-method, cs-uri-stem, http-ver, sc-status, sc-bytes, c-referer, c-user-agent, rs-duration(ms), hit-miss, s-ip\n"; + const string data = "1.2.3.4,4/6/2019 4:00:20 PM +00:00,GET,\"/v3-flatcontainer/microsoft.codeanalysis.common/1.2.2/microsoft.codeanalysis.common.1.2.2.nupkg\",HTTPS,200,2044843,\"NULL\",\"NuGet VS VSIX/4.7.0 (Microsoft Windows NT 10.0.17134.0, VS Enterprise/15.0)\",796,MISS,4.3.2.1"; + + Mock sourceMock = SetupSource(header + data); var writeSucceeded = false; + var outputStream = new MemoryStream(); + Mock destinationMock = SetupDestination(outputStream, () => writeSucceeded = true); + + var collector = new ChinaStatsCollector( + sourceMock.Object, + destinationMock.Object, + Mock.Of>(), + writeHeader: true, + addSourceFilenameColumn: true); + + await collector.TryProcessAsync( + maxFileCount: 10, + fileNameTransform: s => s, + sourceContentType: ContentType.Text, + destinationContentType: ContentType.Text, + CancellationToken.None); + + string[] outputLines = null; + + outputLines = GetStreamLines(outputStream); + Assert.True(writeSucceeded); + Assert.Equal(2, outputLines.Length); + Assert.EndsWith("sourceFilename", outputLines[0]); + Assert.EndsWith("log1", outputLines[1]); + } + + private static Mock SetupDestination(MemoryStream outputStream, Action onSuccess) + { + var destinationMock = new Mock(); destinationMock .Setup(d => d.TryWriteAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((Stream inputStream, Action writeAction, string destinationFileName, ContentType destinationContentType, CancellationToken token) => @@ -146,7 +234,7 @@ public async Task SkipsLinesWithMalformedColumns(string data, int expectedOutput try { writeAction(inputStream, outputStream); - writeSucceeded = true; + onSuccess(); } catch (Exception ex) { @@ -154,21 +242,31 @@ public async Task SkipsLinesWithMalformedColumns(string data, int expectedOutput } return new AsyncOperationResult(true, null); }); + return destinationMock; + } - var collector = new ChinaStatsCollector( - sourceMock.Object, - destinationMock.Object, - Mock.Of>()); + private static Mock SetupSource(string content) + { + var sourceUri = new Uri("https://example.com/log1"); - await collector.TryProcessAsync( - maxFileCount: 10, - fileNameTransform: s => s, - sourceContentType: ContentType.Text, - destinationContentType: ContentType.Text, - CancellationToken.None); + var sourceMock = new Mock(); + sourceMock + .Setup(s => s.GetFilesAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((IEnumerable)new List { sourceUri }); - string[] outputLines = null; + sourceMock + .Setup(s => s.TakeLockAsync(sourceUri, It.IsAny())) + .ReturnsAsync(new AzureBlobLockResult(new Microsoft.WindowsAzure.Storage.Blob.CloudBlob(sourceUri), true, "foo", CancellationToken.None)); + sourceMock + .Setup(s => s.OpenReadAsync(sourceUri, It.IsAny(), It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(content))); + return sourceMock; + } + + private static string[] GetStreamLines(MemoryStream outputStream) + { + string[] outputLines; // need to reopen closed stream var outputBuffer = outputStream.ToArray(); outputStream = new MemoryStream(outputBuffer); @@ -177,11 +275,7 @@ await collector.TryProcessAsync( outputLines = streamReader.ReadToEnd().Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); } - Assert.True(writeSucceeded); - Assert.NotEmpty(outputLines); - Assert.Equal("#Fields: timestamp time-taken c-ip filesize s-ip s-port sc-status sc-bytes cs-method cs-uri-stem - rs-duration rs-bytes c-referrer c-user-agent customer-id x-ec_custom-1", outputLines[0]); - Assert.True(expectedOutputLines <= outputLines.Length - 1); // excluding header + return outputLines; } } } - \ No newline at end of file