diff --git a/eng/publishing/v3/publish-assets.yml b/eng/publishing/v3/publish-assets.yml index beb11af49f1..af61f8bcf80 100644 --- a/eng/publishing/v3/publish-assets.yml +++ b/eng/publishing/v3/publish-assets.yml @@ -22,12 +22,15 @@ jobs: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - name: AzDOBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + - name: AzDOAccount + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildAccount'] ] pool: vmImage: 'windows-2019' steps: - task: DownloadBuildArtifacts@0 displayName: Download Build Assets continueOnError: true + enabled: true inputs: buildType: specific buildVersionToDownload: specific @@ -36,12 +39,9 @@ jobs: buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | - PackageArtifacts/** - BlobArtifacts/** AssetManifests/** PdbArtifacts/** downloadPath: '$(Build.ArtifactStagingDirectory)' - checkDownloadedFiles: true - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' @@ -87,7 +87,12 @@ jobs: /p:MsdlToken=$(microsoft-symbol-server-pat) /p:SymWebToken=$(symweb-symbol-server-pat) /p:BuildQuality='${{ parameters.buildQuality }}' - + /p:AzdoApiToken='$(dn-bot-all-orgs-build-rw-code-rw)' + /p:ArtifactsBasePath='$(Build.ArtifactStagingDirectory)/' + /p:BuildId='$(AzDOBuildId)' + /p:AzureDevOpsOrg='$(AzDOAccount)' + /p:AzureProject='$(AzDOProjectName)' + /p:UseStreamingPublishing='true' - template: /eng/common/templates/steps/publish-logs.yml parameters: StageLabel: '${{ parameters.stageName }}' diff --git a/eng/publishing/v3/setup-maestro-vars.yml b/eng/publishing/v3/setup-maestro-vars.yml index 804e9faee96..ae2e65527b6 100644 --- a/eng/publishing/v3/setup-maestro-vars.yml +++ b/eng/publishing/v3/setup-maestro-vars.yml @@ -38,6 +38,7 @@ jobs: $AzureDevOpsProject = $buildInfo.azureDevOpsProject $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + $AzureDevOpsAccount = $buildInfo.azureDevOpsAccount Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId" Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels" @@ -46,6 +47,7 @@ jobs: Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject" Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId" Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId" + Write-Host "##vso[task.setvariable variable=AzDOBuildAccount;isOutput=true]$AzureDevOpsAccount" } catch { Write-Host $_ diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index 8f24fd48158..28d23ef95c2 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -23,6 +23,10 @@ - SymWebToken : Token used to publish symbol blobs, dll and pdb to Symweb - SymbolPublishingExclusionsFile : Files that have to be excluded from symbol publishing - PdbArtifactsBasePath : Path to dlls and pdbs + - AzdoApiToken : Token used to call the Azure Api to download artifacts and symbols + - ArtifactsBasePath : Staging directory + - UseStreamingPublishing : When set to true it will use Azure Api to download the artifacts and symbols using streaming, + else all the artifacts and symbols are downloaded before publishing. Optional aka.ms link generation parameters: - AkaMSClientId : Client ID for the aka.ms AD application @@ -103,6 +107,8 @@ $(ChecksumsAzureAccountKey) $(InternalChecksumsAzureAccountKey) false + false + $(BlobBasePath) + SymbolsFeedOverride="$(SymbolsFeedOverride)" + AzdoApiToken="$(AzdoApiToken)" + ArtifactsBasePath="$(ArtifactsBasePath)" + BuildId="$(BuildId)" + AzureDevOpsOrg="$(AzureDevOpsOrg)" + AzureProject="$(AzureProject)" + UseStreamingPublishing="$(UseStreamingPublishing)"/> + diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs index 3216cf23dba..926fa188d10 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs @@ -1,8 +1,8 @@ using System.Collections.Generic; using System.IO; -using Castle.DynamicProxy.Generators.Emitters.SimpleAST; using Microsoft.Arcade.Test.Common; using Microsoft.DotNet.Build.Tasks.Feed.Model; +using Microsoft.DotNet.Maestro.Client.Models; using Xunit; namespace Microsoft.DotNet.Build.Tasks.Feed.Tests @@ -85,18 +85,23 @@ public void TemporarySymbolDirectoryDoesNotExists() BuildEngine = buildEngine, }; var path = TestInputs.GetFullPath("Symbol"); - var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", path, false); + var buildAsset = new Dictionary>(); + var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", false, buildAsset, path); Assert.True(task.Log.HasLoggedErrors); } [Fact] public void TemporarySymbolsDirectoryTest() { - var publishTask = new PublishArtifactsInManifestV3(); + var buildEngine = new MockBuildEngine(); + var publishTask = new PublishArtifactsInManifestV3() + { + BuildEngine = buildEngine, + }; var path = TestInputs.GetFullPath("Test"); - publishTask.EnsureTemporarySymbolDirectoryExists(path); + publishTask.EnsureTemporaryDirectoryExists(path); Assert.True(Directory.Exists(path)); - publishTask.DeleteSymbolTemporaryDirectory(path); + publishTask.DeleteTemporaryDirectory(path); Assert.False(Directory.Exists(path)); } @@ -134,6 +139,5 @@ public void PublishSymbolApiIsCalledTest() false, false).IsCompleted); } - } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs index 77d64e62215..791b440ece5 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs @@ -1,23 +1,21 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Build.Framework; using Microsoft.DotNet.Build.CloudTestTasks; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Newtonsoft.Json.Serialization; -using NuGet.Packaging; using NuGet.Packaging.Core; using Sleet; -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.IO; -using System.Linq; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; using MSBuild = Microsoft.Build.Utilities; namespace Microsoft.DotNet.Build.Tasks.Feed @@ -124,17 +122,15 @@ public async Task PushItemsToFeedAsync( return !Log.HasLoggedErrors; } - public async Task PublishToFlatContainerAsync(IEnumerable taskItems, int maxClients, PushOptions pushOptions) + public async Task PublishToFlatContainerAsync(IEnumerable taskItems, int maxClients, + PushOptions pushOptions) { if (taskItems.Any()) { using (var clientThrottle = new SemaphoreSlim(maxClients, maxClients)) { await System.Threading.Tasks.Task.WhenAll(taskItems.Select( - item => - { - return UploadAssetAsync(item, clientThrottle, pushOptions); - } + item => { return UploadAssetAsync(item, pushOptions, clientThrottle); } )); } } @@ -142,8 +138,8 @@ await System.Threading.Tasks.Task.WhenAll(taskItems.Select( public async Task UploadAssetAsync( ITaskItem item, - SemaphoreSlim clientThrottle, - PushOptions options) + PushOptions options, + SemaphoreSlim clientThrottle = null) { string relativeBlobPath = item.GetMetadata("RelativeBlobPath"); @@ -154,58 +150,72 @@ public async Task UploadAssetAsync( relativeBlobPath = $"{recursiveDir}{fileName}"; } - relativeBlobPath = $"{RelativePath}{relativeBlobPath}".Replace("\\", "/"); - - if (relativeBlobPath.Contains("//")) + if (!string.IsNullOrEmpty(relativeBlobPath)) { - Log.LogError( - $"Item '{item.ItemSpec}' RelativeBlobPath contains virtual directory " + - $"without name (double forward slash): '{relativeBlobPath}'"); - return; - } + relativeBlobPath = $"{RelativePath}{relativeBlobPath}".Replace("\\", "/"); - Log.LogMessage($"Uploading {relativeBlobPath}"); + if (relativeBlobPath.StartsWith("//")) + { + Log.LogError( + $"Item '{item.ItemSpec}' RelativeBlobPath contains virtual directory " + + $"without name (double forward slash): '{relativeBlobPath}'"); + return; + } - await clientThrottle.WaitAsync(); + Log.LogMessage($"Uploading {relativeBlobPath}"); - try - { - AzureStorageUtils blobUtils = new AzureStorageUtils(AccountName, AccountKey, ContainerName); + if (clientThrottle != null) + { + await clientThrottle.WaitAsync(); + } - if (!options.AllowOverwrite && await blobUtils.CheckIfBlobExistsAsync(relativeBlobPath)) + try { - if (options.PassIfExistingItemIdentical) + AzureStorageUtils blobUtils = new AzureStorageUtils(AccountName, AccountKey, ContainerName); + + if (!options.AllowOverwrite && await blobUtils.CheckIfBlobExistsAsync(relativeBlobPath)) { - if (!await blobUtils.IsFileIdenticalToBlobAsync(item.ItemSpec, relativeBlobPath)) + if (options.PassIfExistingItemIdentical) { - Log.LogError( - $"Item '{item}' already exists with different contents " + - $"at '{relativeBlobPath}'"); + if (!await blobUtils.IsFileIdenticalToBlobAsync(item.ItemSpec, relativeBlobPath)) + { + Log.LogError( + $"Item '{item}' already exists with different contents " + + $"at '{relativeBlobPath}'"); + } + } + else + { + Log.LogError($"Item '{item}' already exists at '{relativeBlobPath}'"); } } else { - Log.LogError($"Item '{item}' already exists at '{relativeBlobPath}'"); + using (FileStream stream = + new FileStream(item.ItemSpec, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + Log.LogMessage($"Uploading {item} to {relativeBlobPath}."); + await blobUtils.UploadBlockBlobAsync(item.ItemSpec, relativeBlobPath, stream); + } } } - else + catch (Exception exc) { - using (FileStream stream = - new FileStream(item.ItemSpec, FileMode.Open, FileAccess.Read, FileShare.Read)) + Log.LogError( + $"Unable to upload to {relativeBlobPath} in Azure Storage account {AccountName}/{ContainerName} due to {exc}."); + throw; + } + finally + { + if (clientThrottle != null) { - Log.LogMessage($"Uploading {item} to {relativeBlobPath}."); - await blobUtils.UploadBlockBlobAsync(item.ItemSpec, relativeBlobPath, stream); + clientThrottle.Release(); } } } - catch (Exception exc) - { - Log.LogError($"Unable to upload to {relativeBlobPath} in Azure Storage account {AccountName}/{ContainerName} due to {exc}."); - throw; - } - finally + else { - clientThrottle.Release(); + Log.LogError($"Relative blob path is empty."); } } @@ -332,7 +342,7 @@ private ISleetFileSystem GetAzureFileSystem(LocalCache fileCache) } } - private async Task PushAsync(IEnumerable items, PushOptions options) + public async Task PushAsync(IEnumerable items, PushOptions options) { LocalSettings settings = GetSettings(); SleetLogger log = new SleetLogger(Log, NuGet.Common.LogLevel.Verbose); diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs index fef9005240e..0f5be881c36 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs @@ -183,6 +183,21 @@ public string BuildQuality get { return _buildQuality.GetDescription(); } set { Enum.TryParse(value, true, out _buildQuality); } } + public string AzdoApiToken {get; set;} + + public string ArtifactsBasePath { get; set;} + + public string BuildId { get; set; } + + public string AzureProject { get; set; } + + public string AzureDevOpsOrg { get; set; } + + /// + /// If true, uses Azdo Api to download artifacts and symbols files one file at a time during publishing process. + /// If it is set to false, then artifacts and symbols are downloaded in PackageArtifacts and BlobArtifacts directory before publishing. + /// + public bool UseStreamingPublishing { get; set; } = false; /// /// Just an internal flag to keep track whether we published assets via a V3 manifest or not. @@ -372,7 +387,13 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui ChecksumsFeedOverride = this.ChecksumsFeedOverride, ShippingFeedOverride = this.ShippingFeedOverride, TransportFeedOverride = this.TransportFeedOverride, - SymbolsFeedOverride = this.SymbolsFeedOverride + SymbolsFeedOverride = this.SymbolsFeedOverride, + ArtifactsBasePath = this.ArtifactsBasePath, + AzdoApiToken = this.AzdoApiToken, + BuildId = this.BuildId, + AzureProject = this.AzureProject, + AzureDevOpsOrg = this.AzureDevOpsOrg, + UseStreamingPublishing = this.UseStreamingPublishing }; } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 722e9e011f7..2655601a893 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -5,8 +5,10 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; +using System.ComponentModel; using System.IO; using System.Linq; +using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Reflection; @@ -14,11 +16,14 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using Microsoft.Arcade.Common; using Microsoft.Build.Framework; using Microsoft.DotNet.Build.Tasks.Feed.Model; using Microsoft.DotNet.Maestro.Client; using Microsoft.DotNet.Maestro.Client.Models; using Microsoft.DotNet.VersionTools.BuildManifest.Model; +using Newtonsoft.Json; +using NuGet.Packaging; using NuGet.Packaging.Core; using NuGet.Versioning; using static Microsoft.DotNet.Build.Tasks.Feed.GeneralUtils; @@ -103,6 +108,24 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities public string BuildQuality { get; set; } + public string AzdoApiToken { get; set; } + + public string ArtifactsBasePath { get; set; } + + public string AzureDevOpsFeedsApiVersion { get; set; } = "6.0"; + + public string AzureApiVersionForFileDownload { get; set; } = "4.1-preview.4"; + + public string AzureProject { get; set; } + + public string BuildId { get; set; } + + public string AzureDevOpsOrg { get; set; } + + private readonly string AzureDevOpsBaseUrl = $"https://dev.azure.com"; + + public bool UseStreamingPublishing { get; set; } + public readonly Dictionary> FeedConfigs = new Dictionary>(); @@ -126,6 +149,8 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities private const int ExpirationInDays = 3650; + public int TimeoutInMinutes { get; set; } = 5; + protected LatestLinksManager LinkManager { get; set; } = null; /// @@ -138,6 +163,23 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities /// public int RetryDelayMilliseconds { get; set; } = 5000; + public readonly ExponentialRetry RetryHandler = new ExponentialRetry + { + MaxAttempts = 5, + DelayBase = 2.5 // 2.5 ^ 5 = ~1.5 minutes max wait between retries + }; + + private enum ArtifactName + { + [Description("PackageArtifacts")] + PackageArtifacts, + + [Description("BlobArtifacts")] + BlobArtifacts + } + + private int TimeoutInSeconds = 180; + public override bool Execute() { return ExecuteAsync().GetAwaiter().GetResult(); @@ -246,7 +288,6 @@ protected Task PersistPendingAssetLocationAsync(IMaestroApi client) /// /// Protect against accidental publishing of internal assets to non-internal feeds. /// - /// protected void CheckForInternalBuildsOnPublicFeeds(TargetFeedConfig feedConfig) { // If separated out for clarity. @@ -317,7 +358,157 @@ public void CheckForStableAssetsInNonIsolatedFeeds() } /// - /// Publishes symbol, dll and pdb files to symbol server. + /// Publishes files to symbol server(s) one by one using Azure api to download files + /// + /// Path to dll and pdb files + /// Token to authenticate msdl + /// Token to authenticate symweb + /// Right now we do not add any files to this, so this is going to be null + /// If true, the special coreclr module indexed files like DBI, DAC and SOS are published + /// Task + public async Task PublishSymbolsUsingStreamingAsync( + string pdbArtifactsBasePath, + string msdlToken, + string symWebToken, + string symbolPublishingExclusionsFile, + bool publishSpecialClrFiles, + Dictionary> buildAssets) + { + StringBuilder symbolLog = new StringBuilder(); + symbolLog.AppendLine("Publishing Symbols to Symbol server: "); + var symbolCategory = TargetFeedContentType.Symbols; + string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts); + + if (Log.HasLoggedErrors) + { + return; + } + HashSet symbolsToPublish = new HashSet(); + + //Get all the symbol file names + foreach (var asset in buildAssets) + { + var name = asset.Key; + if (name.Contains(".symbols.nupkg")) + { + symbolsToPublish.Add(Path.GetFileName(name)); + } + } + + HashSet feedConfigsForSymbols = FeedConfigs[symbolCategory]; + Dictionary serversToPublish = + GetTargetSymbolServers(feedConfigsForSymbols, msdlToken, symWebToken); + + using (HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true)) + { + foreach (var symbol in symbolsToPublish) + { + string temporarySymbolsDirectory = CreateTemporaryDirectory(); + string localSymbolPath = Path.Combine(temporarySymbolsDirectory, symbol); + symbolLog.AppendLine($"Downloading symbol : {symbol} to {localSymbolPath}"); + + await DownloadFileAsync(client, ArtifactName.BlobArtifacts, containerId, symbol, localSymbolPath); + symbolLog.AppendLine($"Successfully downloaded symbol : {symbol} to {localSymbolPath}"); + List symbolFiles = new List(); + symbolFiles.ToList().Add(localSymbolPath); + + foreach (var server in serversToPublish) + { + var serverPath = server.Key; + var token = server.Value; + symbolLog.AppendLine($"Publishing symbol file {symbol} to {serverPath}:"); + + try + { + await PublishSymbolsHelper.PublishAsync( + Log, + serverPath, + token, + symbolFiles, + null, + null, + ExpirationInDays, + false, + publishSpecialClrFiles, + null, + false, + false, + true); + } + catch (Exception ex) + { + Log.LogError(ex.Message); + } + } + + DeleteTemporaryDirectory(temporarySymbolsDirectory); + } + symbolLog.AppendLine( + $"Performing symbol publishing... \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false "); + symbolLog.AppendLine($"Total number of symbol files : {symbolsToPublish.Count}"); + symbolLog.AppendLine("Successfully published to Symbol Server."); + symbolLog.AppendLine(); + Log.LogMessage(MessageImportance.High, symbolLog.ToString()); + symbolLog.Clear(); + } + + // publishing pdb artifacts + IEnumerable filesToSymbolServer = null; + if (Directory.Exists(pdbArtifactsBasePath)) + { + var pdbEntries = System.IO.Directory.EnumerateFiles( + pdbArtifactsBasePath, + "*.pdb", + System.IO.SearchOption.AllDirectories); + var dllEntries = System.IO.Directory.EnumerateFiles( + pdbArtifactsBasePath, + "*.dll", + System.IO.SearchOption.AllDirectories); + filesToSymbolServer = pdbEntries.Concat(dllEntries); + } + + if (filesToSymbolServer != null && filesToSymbolServer.Any()) + { + foreach (var server in serversToPublish) + { + var serverPath = server.Key; + var token = server.Value; + symbolLog.AppendLine($"Publishing pdbFiles to {serverPath}:"); + + try + { + await PublishSymbolsHelper.PublishAsync( + Log, + serverPath, + token, + null, + filesToSymbolServer, + null, + ExpirationInDays, + false, + publishSpecialClrFiles, + null, + false, + false, + true); + } + catch (Exception ex) + { + Log.LogError(ex.Message); + } + } + + Log.LogMessage(MessageImportance.High, $"Successfully published pdb files"); + } + + else + { + Log.LogMessage(MessageImportance.Low, $"No pdb files to upload to symbol server."); + } + } + + /// + /// Decides how to publish the symbol, dll and pdb files /// /// Path to dll and pdb files /// Token to authenticate msdl @@ -325,20 +516,58 @@ public void CheckForStableAssetsInNonIsolatedFeeds() /// Right now we do not add any files to this, so this is going to be null /// Path to Symbol.nupkgs /// If true, the special coreclr module indexed files like DBI, DAC and SOS are published - /// public async Task HandleSymbolPublishingAsync ( string pdbArtifactsBasePath, string msdlToken, string symWebToken, string symbolPublishingExclusionsFile, - string temporarySymbolsLocation, - bool publishSpecialClrFiles) + bool publishSpecialClrFiles, + Dictionary> buildAssets, + string temporarySymbolsLocation = null) { - StringBuilder symbolLog = new StringBuilder(); - symbolLog.AppendLine("Publishing Symbols to Symbol server: "); + if (UseStreamingPublishing) + { + await PublishSymbolsUsingStreamingAsync( + pdbArtifactsBasePath, + msdlToken, + symWebToken, + symbolPublishingExclusionsFile, + publishSpecialClrFiles, + buildAssets); + } + else + { + await PublishSymbolsfromBlobArtifactsAsync( + pdbArtifactsBasePath, + msdlToken, + symWebToken, + symbolPublishingExclusionsFile, + publishSpecialClrFiles, + temporarySymbolsLocation); + } + } + /// + /// Publishes symbol, dll and pdb files to symbol server. + /// + /// Path to dll and pdb files + /// Token to authenticate msdl + /// Token to authenticate symweb + /// Right now we do not add any files to this, so this is going to be null + /// Path to Symbol.nupkgs + /// If true, the special coreclr module indexed files like DBI, DAC and SOS are published + public async Task PublishSymbolsfromBlobArtifactsAsync( + string pdbArtifactsBasePath, + string msdlToken, + string symWebToken, + string symbolPublishingExclusionsFile, + bool publishSpecialClrFiles, + string temporarySymbolsLocation = null) + { if (Directory.Exists(temporarySymbolsLocation)) { + StringBuilder symbolLog = new StringBuilder(); + symbolLog.AppendLine("Publishing Symbols to Symbol server: "); string[] fileEntries = Directory.GetFiles(temporarySymbolsLocation); var category = TargetFeedContentType.Symbols; @@ -349,10 +578,17 @@ public async Task HandleSymbolPublishingAsync ( GetTargetSymbolServers(feedConfigsForSymbols, msdlToken, symWebToken); IEnumerable filesToSymbolServer = null; + if (Directory.Exists(pdbArtifactsBasePath)) { - var pdbEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.pdb", System.IO.SearchOption.AllDirectories); - var dllEntries = System.IO.Directory.EnumerateFiles(pdbArtifactsBasePath, "*.dll", System.IO.SearchOption.AllDirectories); + var pdbEntries = System.IO.Directory.EnumerateFiles( + pdbArtifactsBasePath, + "*.pdb", + System.IO.SearchOption.AllDirectories); + var dllEntries = System.IO.Directory.EnumerateFiles( + pdbArtifactsBasePath, + "*.dll", + System.IO.SearchOption.AllDirectories); filesToSymbolServer = pdbEntries.Concat(dllEntries); } @@ -363,20 +599,29 @@ public async Task HandleSymbolPublishingAsync ( symbolLog.AppendLine($"Publishing symbol packages to {serverPath}:"); symbolLog.AppendLine( $"Performing symbol publishing...\nSymbolServerPath : ${serverPath} \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false \nTotal number of symbol files : {fileEntries.Length} "); - await PublishSymbolsHelper.PublishAsync( - Log, - serverPath, - token, - fileEntries, - filesToSymbolServer, - null, - ExpirationInDays, - false, - publishSpecialClrFiles, - null, - false, - false, - true); + + try + { + await PublishSymbolsHelper.PublishAsync( + Log, + serverPath, + token, + fileEntries, + filesToSymbolServer, + null, + ExpirationInDays, + false, + publishSpecialClrFiles, + null, + false, + false, + true); + } + catch (Exception ex) + { + Log.LogError(ex.Message); + } + symbolLog.AppendLine("Successfully published to Symbol Server."); symbolLog.AppendLine(); Log.LogMessage(MessageImportance.High, symbolLog.ToString()); @@ -385,7 +630,7 @@ await PublishSymbolsHelper.PublishAsync( } else { - Log.LogError($"Temporary symbols directory {temporarySymbolsLocation} does not exists."); + Log.LogError("Temporary symbols directory does not exists."); } } @@ -395,7 +640,7 @@ await PublishSymbolsHelper.PublishAsync( /// /// /// - /// + /// A map of symbol server path => token to the symbol server public Dictionary GetTargetSymbolServers(HashSet feedConfigsForSymbols, string msdlToken, string symWebToken) { Dictionary serversToPublish = new Dictionary(); @@ -439,19 +684,25 @@ protected async Task HandlePackagePublishingAsync(Dictionary FilterPackages(HashSet + /// Creates Azdo client + /// + /// Name of the org + /// If client is used for downloading the artifact then AutomaticDecompression has to be set, else it is not required + /// Azure devOps project name + /// Default version is 6.0 + /// HttpClient + public HttpClient CreateAzdoClient( + string accountName, + bool setAutomaticDecompression, + string projectName = null, + string versionOverride = null) + { + HttpClientHandler handler = new HttpClientHandler {CheckCertificateRevocationList = true}; + if (setAutomaticDecompression) + { + handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; + } + + string address = $"https://dev.azure.com/{accountName}/"; + + if (!string.IsNullOrEmpty(projectName)) + { + address += $"{projectName}/"; + } + + var client = new HttpClient(handler) + { + BaseAddress = new Uri(address) + }; + client.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", + AzdoApiToken)))); + client.DefaultRequestHeaders.Add( + "Accept", + $"application/xml;api-version={versionOverride ?? AzureDevOpsFeedsApiVersion}"); + return client; + } + + /// + /// Gets the container Id, that is going to be used in another API call to download the assets + /// ContainerId is the same for PackageArtifacts and BlobArtifacts + /// + /// If it is PackageArtifacts or BlobArtifacts + /// ContainerId + private async Task GetContainerIdAsync(ArtifactName artifactName) + { + string uri = + $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/{AzureProject}/_apis/build/builds/{BuildId}/artifacts?api-version={AzureDevOpsFeedsApiVersion}"; + Exception mostRecentlyCaughtException = null; + string containerId = ""; + bool success = await RetryHandler.RunAsync(async attempt => + { + try + { + CancellationTokenSource timeoutTokenSource = + new CancellationTokenSource(TimeSpan.FromMinutes(TimeoutInMinutes)); + + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject); + using HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); + using HttpResponseMessage response = await client.GetAsync(uri, timeoutTokenSource.Token); + + response.EnsureSuccessStatusCode(); + string responseBody = await response.Content.ReadAsStringAsync(); + BuildArtifacts buildArtifacts = JsonConvert.DeserializeObject(responseBody); + + foreach (var artifact in buildArtifacts.value) + { + ArtifactName name; + if (Enum.TryParse(artifact.name, out name) && name == artifactName) + { + string[] segment = artifact.resource.data.Split('/'); + containerId = segment[1]; + return true; + } + } + return false; + } + catch (HttpRequestException toStore) + { + mostRecentlyCaughtException = toStore; + return false; + } + }).ConfigureAwait(false); + + if (string.IsNullOrEmpty(containerId)) + { + Log.LogError("Container Id does not exists"); + } + + if (!success) + { + throw new Exception( + $"Failed to get container id after {RetryHandler.MaxAttempts} attempts. See inner exception for details, {mostRecentlyCaughtException}"); + } + + return containerId; + } + + /// + /// Download artifact file using Azure API + /// + /// Azdo client + /// If it is PackageArtifacts or BlobArtifacts + /// ContainerId where the packageArtifact and BlobArtifacts are stored + /// Name the file we are trying to download + /// Path where the file is being downloaded + private async Task DownloadFileAsync( + HttpClient client, + ArtifactName artifactName, + string containerId, + string fileName, + string path) + { + string uri = + $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/_apis/resources/Containers/{containerId}?itemPath=/{artifactName}/{fileName}&isShallow=true&api-version={AzureApiVersionForFileDownload}"; + Log.LogMessage(MessageImportance.Low, $"Download file uri = {uri}"); + Exception mostRecentlyCaughtException = null; + bool success = await RetryHandler.RunAsync(async attempt => + { + try + { + CancellationTokenSource timeoutTokenSource = + new CancellationTokenSource(TimeSpan.FromMinutes(TimeoutInMinutes)); + + using HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); + using HttpResponseMessage response = await client.GetAsync(uri, timeoutTokenSource.Token); + + response.EnsureSuccessStatusCode(); + + using var fs = new FileStream( + path, + FileMode.Create, + FileAccess.ReadWrite, + FileShare.ReadWrite); + using var stream = await response.Content.ReadAsStreamAsync(); + + try + { + await stream.CopyToAsync(fs); + } + finally + { + stream.Close(); + fs.Close(); + } + + return true; + } + catch (HttpRequestException toStore) + { + mostRecentlyCaughtException = toStore; + return false; + } + }).ConfigureAwait(false); + + if (!success) + { + throw new Exception( + $"Failed to download local file '{path}' after {RetryHandler.MaxAttempts} attempts. See inner exception for details, {mostRecentlyCaughtException}"); + } + } + protected async Task HandleBlobPublishingAsync(Dictionary> buildAssets) { List publishTasks = new List(); @@ -502,19 +919,29 @@ protected async Task HandleBlobPublishingAsync(Dictionary string isolatedString = feedConfig.Isolated ? "Isolated" : "Non-Isolated"; string internalString = feedConfig.Internal ? $", Internal" : ", Public"; string shippingString = blob.NonShipping ? "NonShipping" : "Shipping"; - Log.LogMessage(MessageImportance.High, $"Blob {blob.Id} ({shippingString}) should go to {feedConfig.TargetURL} ({isolatedString}{internalString})"); + Log.LogMessage(MessageImportance.High, + $"Blob {blob.Id} ({shippingString}) should go to {feedConfig.TargetURL} ({isolatedString}{internalString})"); } switch (feedConfig.Type) { case FeedType.AzDoNugetFeed: - publishTasks.Add(PublishBlobsToAzDoNugetFeedAsync(filteredBlobs, buildAssets, feedConfig)); + publishTasks.Add( + PublishBlobsToAzDoNugetFeedAsync( + filteredBlobs, + buildAssets, + feedConfig)); break; case FeedType.AzureStorageFeed: - publishTasks.Add(PublishBlobsToAzureStorageNugetFeedAsync(filteredBlobs, buildAssets, feedConfig)); + publishTasks.Add( + PublishBlobsToAzureStorageNugetFeedAsync( + filteredBlobs, + buildAssets, + feedConfig)); break; default: - Log.LogError($"Unknown target feed type for category '{category}': '{feedConfig.Type}'."); + Log.LogError( + $"Unknown target feed type for category '{category}': '{feedConfig.Type}'."); break; } } @@ -581,7 +1008,7 @@ public void SplitArtifactsInCategories(BuildModel buildModel) } else { - PackagesByCategory[categoryKey] = new HashSet() { packageAsset }; + PackagesByCategory[categoryKey] = new HashSet() {packageAsset}; } } } @@ -608,13 +1035,13 @@ public void SplitArtifactsInCategories(BuildModel buildModel) } else { - BlobsByCategory[categoryKey] = new HashSet() { blobAsset }; + BlobsByCategory[categoryKey] = new HashSet() {blobAsset}; } } } } - private async Task PublishPackagesToAzDoNugetFeedAsync( + private async Task PublishPackagesFromPackageArtifactsToAzDoNugetFeedAsync( HashSet packagesToPublish, Dictionary> buildAssets, TargetFeedConfig feedConfig) @@ -622,17 +1049,108 @@ private async Task PublishPackagesToAzDoNugetFeedAsync( await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClients, async (feed, httpClient, package, feedAccount, feedVisibility, feedName) => { - string localPackagePath = Path.Combine(PackageAssetsBasePath, $"{package.Id}.{package.Version}.nupkg"); + string localPackagePath = + Path.Combine(PackageAssetsBasePath, $"{package.Id}.{package.Version}.nupkg"); if (!File.Exists(localPackagePath)) { Log.LogError($"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); return; } - TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); + TryAddAssetLocation( + package.Id, + package.Version, + buildAssets, + feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed); + + await PushNugetPackageAsync( + feed, + httpClient, + localPackagePath, + package.Id, + package.Version, + feedAccount, + feedVisibility, + feedName); + }); + } + + private async Task PublishPackagesUsingStreamingToAzdoNugetAsync( + HashSet packagesToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + string containerId = await GetContainerIdAsync(ArtifactName.PackageArtifacts); + + if (Log.HasLoggedErrors) + { + return; + } + + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true); + + foreach (var package in packagesToPublish) + { + var packageFilename = $"{package.Id}.{package.Version}.nupkg"; + string temporaryPackageDirectory = + Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryPackageDirectory); + string localPackagePath = Path.Combine(temporaryPackageDirectory, packageFilename); + Log.LogMessage(MessageImportance.Low, $"Downloading package : {packageFilename} to {localPackagePath}"); + + await DownloadFileAsync( + client, + ArtifactName.PackageArtifacts, + containerId, + packageFilename, + localPackagePath); + + if (!File.Exists(localPackagePath)) + { + Log.LogError( + $"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'"); + return; + } + Log.LogMessage(MessageImportance.Low, $"Successfully downloaded package : {packageFilename} to {localPackagePath}"); + + TryAddAssetLocation( + package.Id, + package.Version, + buildAssets, + feedConfig, + AddAssetLocationToAssetAssetLocationType.NugetFeed); - await PushNugetPackageAsync(feed, httpClient, localPackagePath, package.Id, package.Version, feedAccount, feedVisibility, feedName); + using HttpClient httpClient = new HttpClient(new HttpClientHandler + { + CheckCertificateRevocationList = true }); + + httpClient.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String( + Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); + + await PushPackageToNugetFeed(httpClient, feedConfig, localPackagePath, package.Id, package.Version); + + DeleteTemporaryDirectory(localPackagePath); + } + } + + private async Task PublishPackagesToAzDoNugetFeedAsync( + HashSet packagesToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + if (UseStreamingPublishing) + { + await PublishPackagesUsingStreamingToAzdoNugetAsync(packagesToPublish, buildAssets, feedConfig); + } + else + { + await PublishPackagesFromPackageArtifactsToAzDoNugetFeedAsync(packagesToPublish, buildAssets, feedConfig); + } } /// @@ -640,7 +1158,10 @@ await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClien /// /// List of packages to publish /// Information about feed to publish to - public async Task PushNugetPackagesAsync(HashSet packagesToPublish, TargetFeedConfig feedConfig, int maxClients, + public async Task PushNugetPackagesAsync( + HashSet packagesToPublish, + TargetFeedConfig feedConfig, + int maxClients, Func packagePublishAction) { if (!packagesToPublish.Any()) @@ -651,36 +1172,44 @@ public async Task PushNugetPackagesAsync(HashSet packagesToPublish, Target var parsedUri = Regex.Match(feedConfig.TargetURL, PublishingConstants.AzDoNuGetFeedPattern); if (!parsedUri.Success) { - Log.LogError($"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'"); + Log.LogError( + $"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'"); return; } + string feedAccount = parsedUri.Groups["account"].Value; string feedVisibility = parsedUri.Groups["visibility"].Value; string feedName = parsedUri.Groups["feed"].Value; - using (var clientThrottle = new SemaphoreSlim(maxClients, maxClients)) + using var clientThrottle = new SemaphoreSlim(maxClients, maxClients); + + using (HttpClient httpClient = new HttpClient(new HttpClientHandler + {CheckCertificateRevocationList = true})) { - using (HttpClient httpClient = new HttpClient(new HttpClientHandler { CheckCertificateRevocationList = true })) - { - httpClient.Timeout = TimeSpan.FromSeconds(180); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( - "Basic", - Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); + httpClient.Timeout = TimeSpan.FromSeconds(180); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token)))); - await Task.WhenAll(packagesToPublish.Select(async packageToPublish => + await Task.WhenAll(packagesToPublish.Select(async packageToPublish => + { + try { - try - { - // Wait to avoid starting too many processes. - await clientThrottle.WaitAsync(); - await packagePublishAction(feedConfig, httpClient, packageToPublish, feedAccount, feedVisibility, feedName); - } - finally - { - clientThrottle.Release(); - } - })); - } + // Wait to avoid starting too many processes. + await clientThrottle.WaitAsync(); + await packagePublishAction( + feedConfig, + httpClient, + packageToPublish, + feedAccount, + feedVisibility, + feedName); + } + finally + { + clientThrottle.Release(); + } + })); } } @@ -787,7 +1316,7 @@ public async Task PushNugetPackageAsync( } else { - Log.LogMessage(MessageImportance.High, $"Succeeded publishing package '{localPackageLocation}' to feed {feedConfig.TargetURL}"); + Log.LogMessage($"Succeeded publishing package '{localPackageLocation}' to feed {feedConfig.TargetURL}"); } } catch (Exception e) @@ -817,14 +1346,38 @@ private async Task PublishBlobsToAzDoNugetFeedAsync( } else { - Log.LogWarning($"AzDO feed publishing not available for blobs. Blob '{blob.Id}' was not published."); + Log.LogWarning( + $"AzDO feed publishing not available for blobs. Blob '{blob.Id}' was not published."); } } - await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClients, + if (UseStreamingPublishing) + { + await PublishBlobsUsingStreamingToAzDoNugetAsync(packagesToPublish, buildAssets, feedConfig); + } + else + { + await PublishBlobsFromBlobArtifactsToAzDoNugetAsync(packagesToPublish, buildAssets, feedConfig); + } + } + + private async Task PublishBlobsFromBlobArtifactsToAzDoNugetAsync( + HashSet blobsToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + await PushNugetPackagesAsync( + blobsToPublish, + feedConfig, + maxClients: MaxClients, async (feed, httpClient, blob, feedAccount, feedVisibility, feedName) => { - if (TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.Container)) + if (TryAddAssetLocation( + blob.Id, + assetVersion: null, + buildAssets, + feedConfig, + AddAssetLocationToAssetAssetLocationType.Container)) { // Determine the local path to the blob string fileName = Path.GetFileName(blob.Id); @@ -838,54 +1391,229 @@ await PushNugetPackagesAsync(packagesToPublish, feedConfig, m string id; string version; // Determine package ID and version by asking the nuget libraries - using (var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath)) + using (var packageReader = new PackageArchiveReader(localBlobPath)) { PackageIdentity packageIdentity = packageReader.GetIdentity(); id = packageIdentity.Id; version = packageIdentity.Version.ToString(); } - await PushNugetPackageAsync(feed, httpClient, localBlobPath, id, version, feedAccount, feedVisibility, feedName); + await PushNugetPackageAsync( + feed, + httpClient, + localBlobPath, + id, + version, + feedAccount, + feedVisibility, + feedName); } }); } - private async Task PublishPackagesToAzureStorageNugetFeedAsync( - HashSet packagesToPublish, + private async Task PublishBlobsUsingStreamingToAzDoNugetAsync( + HashSet blobsToPublish, Dictionary> buildAssets, TargetFeedConfig feedConfig) { - var packages = packagesToPublish.Select(p => + string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts); + + if (Log.HasLoggedErrors) { - var localPackagePath = Path.Combine(PackageAssetsBasePath, $"{p.Id}.{p.Version}.nupkg"); - if (!File.Exists(localPackagePath)) + return; + } + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true, AzureProject); + + foreach (var blob in blobsToPublish) + { + if (TryAddAssetLocation( + blob.Id, + assetVersion: null, + buildAssets, + feedConfig, + AddAssetLocationToAssetAssetLocationType.Container)) { - Log.LogError($"Could not locate '{p.Id}.{p.Version}' at '{localPackagePath}'"); + string temporaryBlobDirectory = CreateTemporaryDirectory(); + string fileName = Path.GetFileName(blob.Id); + string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); + Log.LogMessage(MessageImportance.Low, $"Downloading blob : {fileName} to {localBlobPath}"); + + await DownloadFileAsync( + client, + ArtifactName.BlobArtifacts, + containerId, + fileName, + localBlobPath); + + if (!File.Exists(localBlobPath)) + { + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + } + Log.LogMessage(MessageImportance.Low, $"Successfully downloaded blob : {fileName} to {localBlobPath}"); + + string id; + string version; + using (var packageReader = new PackageArchiveReader(localBlobPath)) + { + PackageIdentity packageIdentity = packageReader.GetIdentity(); + id = packageIdentity.Id; + version = packageIdentity.Version.ToString(); + } + + await PushBlobToNugetFeed( + feedConfig, + localBlobPath, + id, + version); + + DeleteTemporaryDirectory(temporaryBlobDirectory); } - return localPackagePath; - }); + } + } - if (Log.HasLoggedErrors) + private async Task PushBlobToNugetFeed(TargetFeedConfig feedConfig, string localBlobPath, string id, + string version) + { + var parsedUri = Regex.Match(feedConfig.TargetURL, PublishingConstants.AzDoNuGetFeedPattern); + if (!parsedUri.Success) { + Log.LogError( + $"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'"); return; } + string feedAccount = parsedUri.Groups["account"].Value; + string feedVisibility = parsedUri.Groups["visibility"].Value; + string feedName = parsedUri.Groups["feed"].Value; + + using HttpClient httpClient = new HttpClient(new HttpClientHandler + {CheckCertificateRevocationList = true}); + httpClient.Timeout = TimeSpan.FromSeconds(180); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String( + Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", + feedConfig.Token)))); + try + { + await PushNugetPackageAsync( + feedConfig, + httpClient, + localBlobPath, + id, + version, + feedAccount, + feedVisibility, + feedName); + } + catch (Exception e) + { + Log.LogErrorFromException(e); + } + } - var blobFeedAction = CreateBlobFeedAction(feedConfig); + /// + /// Creates a temporary directory + /// + /// Path to the directory created + public string CreateTemporaryDirectory() + { + string temporaryDirectory = + Path.GetFullPath(Path.Combine(ArtifactsBasePath, Guid.NewGuid().ToString())); + EnsureTemporaryDirectoryExists(temporaryDirectory); + return temporaryDirectory; + } + + private async Task PublishBlobsToAzureStorageNugetFeedAsync( + HashSet blobsToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + if (UseStreamingPublishing) + { + await PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync(blobsToPublish, buildAssets, feedConfig); + } + else + { + await PublishBlobsToAzureStorageNugetAsync(blobsToPublish, buildAssets, feedConfig); + } + + if (LinkManager == null) + { + LinkManager = new LatestLinksManager( + AkaMSClientId, + AkaMSClientSecret, + AkaMSTenant, + AkaMSGroupOwner, + AkaMSCreatedBy, + AkaMsOwners, + Log); + } + // The latest links should be updated only after the publishing is complete, to avoid + // dead links in the interim. + await LinkManager.CreateOrUpdateLatestLinksAsync( + blobsToPublish, + feedConfig, + PublishingConstants.ExpectedFeedUrlSuffix.Length); + } + private async Task PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync( + HashSet blobsToPublish, + Dictionary> buildAssets, + TargetFeedConfig feedConfig) + { + string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts); + + if (Log.HasLoggedErrors) + { + return; + } + var blobFeedAction = CreateBlobFeedAction(feedConfig); var pushOptions = new PushOptions { AllowOverwrite = feedConfig.AllowOverwrite, PassIfExistingItemIdentical = true }; + using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true, AzureProject); - packagesToPublish - .ToList() - .ForEach(package => TryAddAssetLocation(package.Id, package.Version, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed)); + foreach (var blob in blobsToPublish) + { + string temporaryBlobDirectory = CreateTemporaryDirectory(); + var fileName = Path.GetFileName(blob.Id); + var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); + Log.LogMessage(MessageImportance.Low, $"Downloading blob : {fileName} to {localBlobPath}"); + + await DownloadFileAsync( + client, + ArtifactName.BlobArtifacts, + containerId, + fileName, + localBlobPath); + + if (!File.Exists(localBlobPath)) + { + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + } + Log.LogMessage(MessageImportance.Low, $"Successfully downloaded blob : {fileName} to {localBlobPath}"); - await blobFeedAction.PushToFeedAsync(packages, pushOptions); + var item = new Microsoft.Build.Utilities.TaskItem(localBlobPath, + new Dictionary + { + {"RelativeBlobPath", blob.Id} + }); + + TryAddAssetLocation( + blob.Id, + assetVersion: null, + buildAssets, + feedConfig, + AddAssetLocationToAssetAssetLocationType.Container); + + await blobFeedAction.UploadAssetAsync(item, pushOptions, null); + DeleteTemporaryDirectory(temporaryBlobDirectory); + } } - private async Task PublishBlobsToAzureStorageNugetFeedAsync( + private async Task PublishBlobsToAzureStorageNugetAsync( HashSet blobsToPublish, Dictionary> buildAssets, TargetFeedConfig feedConfig) @@ -921,18 +1649,14 @@ private async Task PublishBlobsToAzureStorageNugetFeedAsync( blobsToPublish .ToList() - .ForEach(blob => TryAddAssetLocation(blob.Id, assetVersion: null, buildAssets, feedConfig, AddAssetLocationToAssetAssetLocationType.Container)); + .ForEach(blob => TryAddAssetLocation( + blob.Id, + assetVersion: null, + buildAssets, + feedConfig, + AddAssetLocationToAssetAssetLocationType.Container)); await blobFeedAction.PublishToFlatContainerAsync(blobs, maxClients: MaxClients, pushOptions); - - if (LinkManager == null) - { - LinkManager = new LatestLinksManager(AkaMSClientId, AkaMSClientSecret, AkaMSTenant, AkaMSGroupOwner, AkaMSCreatedBy, AkaMsOwners, Log); - } - - // The latest links should be updated only after the publishing is complete, to avoid - // dead links in the interim. - await LinkManager.CreateOrUpdateLatestLinksAsync(blobsToPublish, feedConfig, PublishingConstants.ExpectedFeedUrlSuffix.Length); } private BlobFeedAction CreateBlobFeedAction(TargetFeedConfig feedConfig) @@ -974,6 +1698,102 @@ private BlobFeedAction CreateBlobFeedAction(TargetFeedConfig feedConfig) } } + private async Task PushPackageToNugetFeed( + HttpClient httpClient, + TargetFeedConfig feedConfig, + string localPackagePath, + string id, + string version) + { + var parsedUri = Regex.Match(feedConfig.TargetURL, PublishingConstants.AzDoNuGetFeedPattern); + + if (!parsedUri.Success) + { + Log.LogError( + $"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'"); + return; + } + + string feedAccount = parsedUri.Groups["account"].Value; + string feedVisibility = parsedUri.Groups["visibility"].Value; + string feedName = parsedUri.Groups["feed"].Value; + + await PushNugetPackageAsync( + feedConfig, + httpClient, + localPackagePath, + id, + version, + feedAccount, + feedVisibility, + feedName); + } + + /// + /// Create Temporary directory if it does not exists. + /// + /// + public void EnsureTemporaryDirectoryExists(string temporaryLocation) + { + if (!Directory.Exists(temporaryLocation)) + { + Directory.CreateDirectory(temporaryLocation); + } + } + + /// + /// Delete the files after publishing, this is part of cleanup + /// + /// + public void DeleteTemporaryFiles(string temporaryLocation) + { + try + { + if (Directory.Exists(temporaryLocation)) + { + string[] fileEntries = Directory.GetFiles(temporaryLocation); + foreach (var file in fileEntries) + { + File.Delete(file); + } + } + } + catch (Exception ex) + { + Log.LogWarning(ex.Message); + } + } + + /// + /// Deletes the temporary folder, this is part of clean up + /// + /// + public void DeleteTemporaryDirectory(string temporaryLocation) + { + var attempts = 0; + if (Directory.Exists(temporaryLocation)) + { + do + { + try + { + attempts++; + Log.LogMessage(MessageImportance.Low, $"Deleting directory : {temporaryLocation}"); + Directory.Delete(temporaryLocation, true); + break; + } + catch (Exception ex) + { + if (attempts == MaxRetryCount) + Log.LogMessage(MessageImportance.Low, $"Unable to delete the directory because of {ex.Message} after {attempts} attempts."); + } + Log.LogMessage(MessageImportance.Low, $"Retrying to delete {temporaryLocation}, attempt number {attempts}"); + Task.Delay(RetryDelayMilliseconds).Wait(); + } + while (true); + } + } + protected bool AnyMissingRequiredProperty() { foreach (PropertyInfo prop in this.GetType().GetProperties()) @@ -1024,6 +1844,15 @@ protected bool AnyMissingRequiredBaseProperties() Log.LogError($"The property {nameof(BuildAssetRegistryToken)} is required but doesn't have a value set."); } + if (string.IsNullOrEmpty(AzdoApiToken)) + { + Log.LogError($"The property {nameof(AzdoApiToken)} is required but doesn't have a value set."); + } + + if (string.IsNullOrEmpty(ArtifactsBasePath)) + { + Log.LogError($"The property {nameof(ArtifactsBasePath)} is required but doesn't have a value set."); + } return Log.HasLoggedErrors; } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs index 1aa5bb834ed..4228c42d701 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs @@ -1,16 +1,16 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.Build.Framework; -using Microsoft.DotNet.Build.Tasks.Feed.Model; -using Microsoft.DotNet.Maestro.Client; -using Microsoft.DotNet.Maestro.Client.Models; -using Microsoft.DotNet.VersionTools.BuildManifest.Model; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; +using Microsoft.Build.Framework; +using Microsoft.DotNet.Build.Tasks.Feed.Model; +using Microsoft.DotNet.Maestro.Client; +using Microsoft.DotNet.Maestro.Client.Models; +using Microsoft.DotNet.VersionTools.BuildManifest.Model; namespace Microsoft.DotNet.Build.Tasks.Feed { @@ -98,17 +98,7 @@ public override async Task ExecuteAsync() return false; } - string temporarySymbolsLocation = - Path.GetFullPath(Path.Combine(BlobAssetsBasePath, @"..\", "tempSymbols")); - - EnsureTemporarySymbolDirectoryExists(temporarySymbolsLocation); - SplitArtifactsInCategories(BuildModel); - DeleteSymbolTemporaryFiles(temporarySymbolsLocation); - - //Copying symbol files to temporary location is required because the symUploader API needs read/write access to the files, - //since we publish blobs and symbols in parallel this will cause IO exceptions. - CopySymbolFilesToTemporaryLocation(BuildModel, temporarySymbolsLocation); if (Log.HasLoggedErrors) { @@ -199,16 +189,36 @@ public override async Task ExecuteAsync() return false; } + string temporarySymbolsLocation = ""; + if (!UseStreamingPublishing) + { + temporarySymbolsLocation = + Path.GetFullPath(Path.Combine(BlobAssetsBasePath, @"..\", "tempSymbols")); + + EnsureTemporaryDirectoryExists(temporarySymbolsLocation); + DeleteTemporaryFiles(temporarySymbolsLocation); + + // Copying symbol files to temporary location is required because the symUploader API needs read/write access to the files, + // since we publish blobs and symbols in parallel this will cause IO exceptions. + CopySymbolFilesToTemporaryLocation(BuildModel, temporarySymbolsLocation); + } + await Task.WhenAll(new Task[] { HandlePackagePublishingAsync(buildAssets), HandleBlobPublishingAsync(buildAssets), - HandleSymbolPublishingAsync(PdbArtifactsBasePath, MsdlToken, - SymWebToken, SymbolPublishingExclusionsFile, temporarySymbolsLocation, PublishSpecialClrFiles) + HandleSymbolPublishingAsync( + PdbArtifactsBasePath, + MsdlToken, + SymWebToken, + SymbolPublishingExclusionsFile, + PublishSpecialClrFiles, + buildAssets, + temporarySymbolsLocation) }); - DeleteSymbolTemporaryFiles(temporarySymbolsLocation); - DeleteSymbolTemporaryDirectory(temporarySymbolsLocation); - Log.LogMessage(MessageImportance.High, "Successfully deleted the temporary symbols directory."); + DeleteTemporaryFiles(temporarySymbolsLocation); + DeleteTemporaryDirectory(temporarySymbolsLocation); + await PersistPendingAssetLocationAsync(client); } catch (Exception e) @@ -245,52 +255,6 @@ private void CopySymbolFilesToTemporaryLocation(BuildModel buildModel, string sy } } - /// - /// Create Temporary Symbols directory if it does not exists. - /// - /// - public void EnsureTemporarySymbolDirectoryExists(string temporarySymbolsLocation) - { - if (!Directory.Exists(temporarySymbolsLocation)) - { - Directory.CreateDirectory(temporarySymbolsLocation); - } - } - /// - /// Delete the symbols files after publishing to Symbol server(s), this is part of cleanup - /// - /// - public void DeleteSymbolTemporaryFiles(string temporarySymbolsLocation) - { - try - { - if (Directory.Exists(temporarySymbolsLocation)) - { - string[] fileEntries = Directory.GetFiles(temporarySymbolsLocation); - foreach (var file in fileEntries) - { - File.Delete(file); - } - } - } - catch (Exception ex) - { - Log.LogWarning(ex.Message); - } - } - - /// - /// Deletes the temporary symbol folder, this is part of clean up - /// - /// - public void DeleteSymbolTemporaryDirectory(string temporarySymbolLocation) - { - if (Directory.Exists(temporarySymbolLocation)) - { - Directory.Delete(temporarySymbolLocation); - } - } - public string GetFeed(string feed, string feedOverride) { return (AllowFeedOverrides && !string.IsNullOrEmpty(feedOverride)) ? feedOverride : feed; diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/GeneralUtils.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/GeneralUtils.cs index 90b58e851a3..3e039ac74b8 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/GeneralUtils.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/GeneralUtils.cs @@ -134,13 +134,13 @@ public static async Task CompareStreamsAsync(Stream localFileStream, Strea /// the streams make no guarantee that they will return a full block each time when read operations are performed, so we /// must be sure to only compare the minimum number of bytes returned. /// - public static Task CompareLocalPackageToFeedPackage( + public static async Task CompareLocalPackageToFeedPackage( string localPackageFullPath, string packageContentUrl, HttpClient client, TaskLoggingHelper log) { - return CompareLocalPackageToFeedPackage( + return await CompareLocalPackageToFeedPackage( localPackageFullPath, packageContentUrl, client, diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/BuildArtifacts.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/BuildArtifacts.cs new file mode 100644 index 00000000000..16df412ab49 --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/BuildArtifacts.cs @@ -0,0 +1,35 @@ +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace Microsoft.DotNet.Build.Tasks.Feed +{ + public class Properties + { + public string artifactsize { get; set; } + public string RootId { get; set; } + public string localpath { get; set; } + } + + public class Resource + { + public string type { get; set; } + public string data { get; set; } + public Properties properties { get; set; } + public string url { get; set; } + public string downloadUrl { get; set; } + } + + public class Value + { + public int id { get; set; } + public string name { get; set; } + public string source { get; set; } + public Resource resource { get; set; } + } + + public class BuildArtifacts + { + public int count { get; set; } + public IList value { get; set; } + } +}