Support building with missing workload packs

src/Microsoft.DotNet.Build.Tasks.Workloads/src/CreateVisualStudioWorkload.wix.cs

@@ -11,6 +11,7 @@
 using Microsoft.DotNet.Build.Tasks.Workloads.Msi;
 using Microsoft.DotNet.Build.Tasks.Workloads.Swix;
 using Microsoft.NET.Sdk.WorkloadManifestReader;
+using static Microsoft.DotNet.Build.Tasks.Workloads.Msi.WorkloadManifestMsi;
 using Parallel = System.Threading.Tasks.Parallel;
 
 namespace Microsoft.DotNet.Build.Tasks.Workloads
@@ -173,6 +174,15 @@ public bool DisableParallelPackageGroupProcessing
             set;
         }
 
+        /// <summary>
+        /// Allow VS workload generation to proceed if any nupkgs declared in the manifest are not found on disk.
+        /// </summary>
+        public bool AllowMissingPacks
+        {
+            get;
+            set;
+        } = false;
+
         public override bool Execute()
         {
             try
@@ -213,21 +223,54 @@ public override bool Execute()
                     //    ensuring that the pack and MSI is only generated once.
                     WorkloadManifest manifest = manifestPackage.GetManifest();
 
+                    List<WorkloadPackGroupJson> packGroupJsonList = new();
+
                     foreach (WorkloadDefinition workload in manifest.Workloads.Values)
                     {
                         if ((workload is WorkloadDefinition wd) && (wd.Platforms == null || wd.Platforms.Any(platform => platform.StartsWith("win"))) && (wd.Packs != null))
                         {
                             Dictionary<string, List<WorkloadPackPackage>> packsInWorkloadByPlatform = new();
 
+                            string packGroupId = null;
+                            WorkloadPackGroupJson packGroupJson = null;
+                            if (CreateWorkloadPackGroups)
+                            {
+                                packGroupId = WorkloadPackGroupPackage.GetPackGroupID(workload.Id);
+                                packGroupJson = new WorkloadPackGroupJson()
+                                {
+                                    GroupPackageId = packGroupId,
+                                    GroupPackageVersion = manifestPackage.PackageVersion.ToString()
+                                };
+                                packGroupJsonList.Add(packGroupJson);
+                            }
+
+
                             foreach (WorkloadPackId packId in wd.Packs)
                             {
                                 WorkloadPack pack = manifest.Packs[packId];
 
+                                if (CreateWorkloadPackGroups)
+                                {
+                                    packGroupJson.Packs.Add(new WorkloadPackJson()
+                                    {
+                                        PackId = pack.Id,
+                                        PackVersion = pack.Version
+                                    });
+                                }
+
                                 foreach ((string sourcePackage, string[] platforms) in WorkloadPackPackage.GetSourcePackages(PackageSource, pack))
                                 {
                                     if (!File.Exists(sourcePackage))
                                     {
-                                        throw new FileNotFoundException(message: null, fileName: sourcePackage);
+                                        if (AllowMissingPacks)
+                                        {
+                                            Log.LogMessage($"Pack {sourcePackage} - {string.Join(",", platforms)} could not be found, it will be skipped.");
+                                            continue;
+                                        }
+                                        else
+                                        {
+                                            throw new FileNotFoundException(message: "NuGet package not found", fileName: sourcePackage);
+                                        }
                                     }
 
                                     // Create new build data and add the pack if we haven't seen it previously.
@@ -264,7 +307,7 @@ public override bool Execute()
                                 }
                             }
 
-                            string packGroupId = null;
+
 
                             if (CreateWorkloadPackGroups)
                             {
@@ -278,7 +321,6 @@ public override bool Execute()
                                     if (!packGroupPackages.TryGetValue(uniquePackGroupKey, out var groupPackage))
                                     {
                                         groupPackage = new WorkloadPackGroupPackage(workload.Id);
-                                        packGroupId = groupPackage.Id;
                                         groupPackage.Packs.AddRange(kvp.Value);
                                         packGroupPackages[uniquePackGroupKey] = groupPackage;
                                     }
@@ -288,8 +330,11 @@ public override bool Execute()
                                         groupPackage.ManifestsPerPlatform[platform] = new();
                                     }
                                     groupPackage.ManifestsPerPlatform[platform].Add(manifestPackage);
+                                }
 
-                                    manifestMsisByPlatform[platform].WorkloadPackGroups.Add(groupPackage);
+                                foreach (var manifestMsi in manifestMsisByPlatform.Values)
+                                {
+                                    manifestMsi.WorkloadPackGroups.AddRange(packGroupJsonList);
                                 }
                             }
 
@@ -316,11 +361,8 @@ public override bool Execute()
                         WorkloadPackMsi msi = new(data.Package, platform, BuildEngine, WixToolsetPath, BaseIntermediateOutputPath);
                         ITaskItem msiOutputItem = msi.Build(MsiOutputPath, IceSuppressions);
 
-                        // Create the JSON manifest for CLI based installations.
-                        string msiJsonPath = MsiProperties.Create(msiOutputItem.ItemSpec);
-
                         // Generate a .csproj to package the MSI and its manifest for CLI installs.
-                        MsiPayloadPackageProject csproj = new(msi.Metadata, msiOutputItem, BaseIntermediateOutputPath, BaseOutputPath, Path.GetFullPath(msiJsonPath));
+                        MsiPayloadPackageProject csproj = new(msi.Metadata, msiOutputItem, BaseIntermediateOutputPath, BaseOutputPath, msi.NuGetPackageFiles);
                         msiOutputItem.SetMetadata(Metadata.PackageProject, csproj.Create());
 
                         lock (msiItems)
@@ -359,11 +401,8 @@ public override bool Execute()
                         WorkloadPackGroupMsi msi = new(packGroup, platform, BuildEngine, WixToolsetPath, BaseIntermediateOutputPath);
                         ITaskItem msiOutputItem = msi.Build(MsiOutputPath, IceSuppressions);
 
-                        // Create the JSON manifest for CLI based installations.
-                        string msiJsonPath = MsiProperties.Create(msiOutputItem.ItemSpec);
-
                         // Generate a .csproj to package the MSI and its manifest for CLI installs.
-                        MsiPayloadPackageProject csproj = new(msi.Metadata, msiOutputItem, BaseIntermediateOutputPath, BaseOutputPath, Path.GetFullPath(msiJsonPath));
+                        MsiPayloadPackageProject csproj = new(msi.Metadata, msiOutputItem, BaseIntermediateOutputPath, BaseOutputPath, msi.NuGetPackageFiles);
                         msiOutputItem.SetMetadata(Metadata.PackageProject, csproj.Create());
 
                         lock (msiItems)
@@ -397,9 +436,6 @@ public override bool Execute()
                 {
                     ITaskItem msiOutputItem = msi.Build(MsiOutputPath, IceSuppressions);
 
-                    // Create the JSON manifest for CLI based installations.
-                    string msiJsonPath = MsiProperties.Create(msiOutputItem.ItemSpec);
-
                     // Generate SWIX authoring for the MSI package.
                     MsiSwixProject swixProject = new(msiOutputItem, BaseIntermediateOutputPath, BaseOutputPath);
                     ITaskItem swixProjectItem = new TaskItem(swixProject.Create());
@@ -411,7 +447,7 @@ public override bool Execute()
                     }
 
                     // Generate a .csproj to package the MSI and its manifest for CLI installs.
-                    MsiPayloadPackageProject csproj = new(msi.Metadata, msiOutputItem, BaseIntermediateOutputPath, BaseOutputPath, Path.GetFullPath(msiJsonPath));
+                    MsiPayloadPackageProject csproj = new(msi.Metadata, msiOutputItem, BaseIntermediateOutputPath, BaseOutputPath, msi.NuGetPackageFiles);
                     msiOutputItem.SetMetadata(Metadata.PackageProject, csproj.Create());
 
                     lock (msiItems)

src/Microsoft.DotNet.Build.Tasks.Workloads/src/Misc/msi.csproj

@@ -24,9 +24,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <None Include="__MSI__" Pack="true" PackagePath="\data" />
-    <None Include="__MSI_JSON__" Pack="true" PackagePath="\data\msi.json" />
-    <None Include="__LICENSE_FILENAME__" Pack="true" PackagePath="\" />
   </ItemGroup>
 
   <Target Name="AddPackageIcon"

src/Microsoft.DotNet.Build.Tasks.Workloads/src/Msi/MsiBase.wix.cs

@@ -4,6 +4,7 @@
 #nullable enable
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using Microsoft.Build.Framework;
@@ -97,6 +98,8 @@ protected string WixToolsetPath
             get;
         }
 
+        public Dictionary<string, string> NuGetPackageFiles { get; set; } = new();
+
         public MsiBase(MsiMetadata metadata, IBuildEngine buildEngine, string wixToolsetPath, 
             string platform, string baseIntermediateOutputPath)
         {
@@ -217,6 +220,17 @@ protected ITaskItem Link(string compilerOutputPath, string outputFile, ITaskItem
 
             return msiItem;
         }
+
+        protected void AddDefaultPackageFiles(ITaskItem msi)
+        {
+            NuGetPackageFiles[msi.GetMetadata(Workloads.Metadata.FullPath)] = @"\data";
+
+            // Create the JSON manifest for CLI based installations.
+            string msiJsonPath = MsiProperties.Create(msi.ItemSpec);
+            NuGetPackageFiles[Path.GetFullPath(msiJsonPath)] = "\\data\\msi.json";
+
+            NuGetPackageFiles["LICENSE.TXT"] = @"\";
+        }
     }
 }
 

src/Microsoft.DotNet.Build.Tasks.Workloads/src/Msi/MsiPayloadPackageProject.wix.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Text;
+using System.Xml.Linq;
 using Microsoft.Build.Framework;
 
 namespace Microsoft.DotNet.Build.Tasks.Workloads.Msi
@@ -26,22 +27,24 @@ protected override string ProjectFile
             get;
         }
 
-        public MsiPayloadPackageProject(MsiMetadata package, ITaskItem msi, string baseIntermediateOutputPath, string baseOutputPath, string msiJsonPath) :
+        //  Key: path to file, value: path in package
+        public Dictionary<string, string> PackageContents { get; set; } = new();
+
+        public MsiPayloadPackageProject(MsiMetadata package, ITaskItem msi, string baseIntermediateOutputPath, string baseOutputPath, Dictionary<string, string> packageContents) :
             base(baseIntermediateOutputPath, baseOutputPath)
         {
             string platform = msi.GetMetadata(Metadata.Platform);
             ProjectSourceDirectory = Path.Combine(SourceDirectory, "msiPackage", platform, package.Id);
             ProjectFile = "msi.csproj";
 
+            PackageContents = packageContents;
+
             ReplacementTokens[PayloadPackageTokens.__AUTHORS__] = package.Authors;
             ReplacementTokens[PayloadPackageTokens.__COPYRIGHT__] = package.Copyright;
             ReplacementTokens[PayloadPackageTokens.__DESCRIPTION__] = package.Description;
             ReplacementTokens[PayloadPackageTokens.__PACKAGE_ID__] = $"{package.Id}.Msi.{platform}";
             ReplacementTokens[PayloadPackageTokens.__PACKAGE_PROJECT_URL__] = package.ProjectUrl;
             ReplacementTokens[PayloadPackageTokens.__PACKAGE_VERSION__] = $"{package.PackageVersion}";
-            ReplacementTokens[PayloadPackageTokens.__MSI__] = msi.GetMetadata(Metadata.FullPath);
-            ReplacementTokens[PayloadPackageTokens.__MSI_JSON__] = msiJsonPath;
-            ReplacementTokens[PayloadPackageTokens.__LICENSE_FILENAME__] = "LICENSE.TXT";
         }
 
         /// <inheritdoc />
@@ -50,6 +53,18 @@ public override string Create()
             string msiCsproj = EmbeddedTemplates.Extract("msi.csproj", ProjectSourceDirectory);
 
             Utils.StringReplace(msiCsproj, ReplacementTokens, Encoding.UTF8);
+
+            var proj = XDocument.Load(msiCsproj);
+            var itemGroup = proj.Root.Element("ItemGroup");
+            foreach (var packageFile in PackageContents)
+            {
+                itemGroup.Add(new XElement("None",
+                    new XAttribute("Include", packageFile.Key),
+                    new XAttribute("Pack", "true"),
+                    new XAttribute("PackagePath", packageFile.Value)));
+            }
+            proj.Save(msiCsproj);
+
             EmbeddedTemplates.Extract("Icon.png", ProjectSourceDirectory);
             EmbeddedTemplates.Extract("LICENSE.TXT", ProjectSourceDirectory);
 

src/Microsoft.DotNet.Build.Tasks.Workloads/src/Msi/PayloadPackageTokens.wix.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Text;
 
@@ -12,9 +12,6 @@ internal static class PayloadPackageTokens
         public static readonly string __AUTHORS__ = nameof(__AUTHORS__);
         public static readonly string __COPYRIGHT__ = nameof(__COPYRIGHT__);
         public static readonly string __DESCRIPTION__ = nameof(__DESCRIPTION__);
-        public static readonly string __LICENSE_FILENAME__ = nameof(__LICENSE_FILENAME__);
-        public static readonly string __MSI__ = nameof(__MSI__);
-        public static readonly string __MSI_JSON__ = nameof(__MSI_JSON__);
         public static readonly string __PACKAGE_ID__ = nameof(__PACKAGE_ID__);
         public static readonly string __PACKAGE_PROJECT_URL__ = nameof(__PACKAGE_PROJECT_URL__);
         public static readonly string __PACKAGE_VERSION__ = nameof(__PACKAGE_VERSION__);

src/Microsoft.DotNet.Build.Tasks.Workloads/src/Msi/WorkloadManifestMsi.wix.cs

@@ -25,7 +25,8 @@ internal class WorkloadManifestMsi : MsiBase
         /// </summary>
         private static readonly string ManifestIdDirectory = "ManifestIdDir";
 
-        public List<WorkloadPackGroupPackage> WorkloadPackGroups { get; } = new();
+        public List<WorkloadPackGroupJson> WorkloadPackGroups { get; } = new();
+
 
         public WorkloadManifestMsi(WorkloadManifestPackage package, string platform, IBuildEngine buildEngine, string wixToolsetPath,
             string baseIntermediateOutputPath) : 
@@ -55,6 +56,11 @@ public override ITaskItem Build(string outputPath, ITaskItem[]? iceSuppressions
                 throw new Exception(Strings.HeatFailedToHarvest);
             }
 
+            foreach (var file in Directory.GetFiles(packageDataDirectory).Select(f => Path.GetFullPath(f)))
+            {
+                NuGetPackageFiles[file] = @"\data\extractedManifest\" + Path.GetFileName(file);
+            }
+
             //  Add WorkloadPackGroups.json to add to workload manifest MSI
             string? jsonContentWxs = null;
             string? jsonDirectory = null;
@@ -63,27 +69,12 @@ public override ITaskItem Build(string outputPath, ITaskItem[]? iceSuppressions
             {
                 jsonContentWxs = Path.Combine(WixSourceDirectory, "JsonContent.wxs");
 
-                List<WorkloadPackGroupJson> packGroupListJson = new List<WorkloadPackGroupJson>();
-                foreach (var packGroup in WorkloadPackGroups)
-                {
-                    var json = new WorkloadPackGroupJson()
-                    {
-                        GroupPackageId = packGroup.Id,
-                        GroupPackageVersion = packGroup.GetMsiMetadata().PackageVersion.ToString()
-                    };
-                    json.Packs.AddRange(packGroup.Packs.Select(p => new WorkloadPackJson()
-                    {
-                        PackId = p.Id,
-                        PackVersion = p.PackageVersion.ToString()
-                    }));
-
-                    packGroupListJson.Add(json);
-                }
-
-                string jsonAsString = JsonSerializer.Serialize(packGroupListJson, typeof(IList<WorkloadPackGroupJson>), new JsonSerializerOptions() { WriteIndented = true });
+                string jsonAsString = JsonSerializer.Serialize(WorkloadPackGroups, typeof(IList<WorkloadPackGroupJson>), new JsonSerializerOptions() { WriteIndented = true });
                 jsonDirectory = Path.Combine(WixSourceDirectory, "json");
                 Directory.CreateDirectory(jsonDirectory);
-                File.WriteAllText(Path.Combine(jsonDirectory, "WorkloadPackGroups.json"), jsonAsString);
+
+                string jsonFullPath = Path.GetFullPath(Path.Combine(jsonDirectory, "WorkloadPackGroups.json"));
+                File.WriteAllText(jsonFullPath, jsonAsString);
 
                 HarvesterToolTask jsonHeat = new(BuildEngine, WixToolsetPath)
                 {
@@ -99,6 +90,8 @@ public override ITaskItem Build(string outputPath, ITaskItem[]? iceSuppressions
                 {
                     throw new Exception(Strings.HeatFailedToHarvest);
                 }
+
+                NuGetPackageFiles[jsonFullPath] = @"\data\extractedManifest\" + Path.GetFileName(jsonFullPath);
             }
 
             CompilerToolTask candle = CreateDefaultCompiler();
@@ -146,20 +139,22 @@ public override ITaskItem Build(string outputPath, ITaskItem[]? iceSuppressions
             ITaskItem msi = Link(candle.OutputPath, 
                 Path.Combine(outputPath, Path.GetFileNameWithoutExtension(Package.PackagePath) + $"-{Platform}.msi"),
                 iceSuppressions);
+
+            AddDefaultPackageFiles(msi);
 
             return msi;
         }
 
 
-        class WorkloadPackGroupJson
+        public class WorkloadPackGroupJson
         {
             public string? GroupPackageId { get; set; }
             public string? GroupPackageVersion { get; set; }
 
             public List<WorkloadPackJson> Packs { get; set; } = new List<WorkloadPackJson>();
         }
 
-        class WorkloadPackJson
+        public class WorkloadPackJson
         {
             public string? PackId { get; set; }
 

src/Microsoft.DotNet.Build.Tasks.Workloads/src/Msi/WorkloadPackGroupMsi.wix.cs

@@ -143,6 +143,8 @@ public override ITaskItem Build(string outputPath, ITaskItem[] iceSuppressions)
 
             ITaskItem msi = Link(candle.OutputPath, msiFileName, iceSuppressions);
 
+            AddDefaultPackageFiles(msi);
+
             return msi;
 
         }

src/Microsoft.DotNet.Build.Tasks.Workloads/src/Msi/WorkloadPackMsi.wix.cs

@@ -78,6 +78,8 @@ public override ITaskItem Build(string outputPath, ITaskItem[]? iceSuppressions
 
             ITaskItem msi = Link(candle.OutputPath, msiFileName, iceSuppressions);
 
+            AddDefaultPackageFiles(msi);
+
             return msi;
         }
 

src/Microsoft.DotNet.Build.Tasks.Workloads/src/WorkloadPackGroupPackage.wix.cs

@@ -23,7 +23,12 @@ internal class WorkloadPackGroupPackage
         public WorkloadPackGroupPackage(string workloadName)
         {
             WorkloadName = workloadName;
-            Id = Utils.ToSafeId(workloadName) + ".WorkloadPacks";
+            Id = GetPackGroupID(workloadName);
+        }
+
+        public static string GetPackGroupID(string workloadName)
+        {
+            return Utils.ToSafeId(workloadName) + ".WorkloadPacks";
         }
 
         public MsiMetadata GetMsiMetadata()