Auto-populate package source mappings for darc-* feeds

src/Microsoft.DotNet.Darc/DarcLib/Helpers/DependencyFileManager.cs

@@ -550,7 +550,7 @@ public async Task<GitFileContentContainer> UpdateDependencyFiles(
         // At this point we only care about the Maestro managed locations for the assets.
         // Flatten the dictionary into a set that has all the managed feeds
         Dictionary<string, HashSet<string>> managedFeeds = FlattenLocationsAndSplitIntoGroups(itemsToUpdateLocations);
-        nugetConfig = UpdatePackageSources(nugetConfig, managedFeeds);
+        nugetConfig = UpdatePackageSources(nugetConfig, managedFeeds, itemsToUpdateLocations);
 
         // Update the dotnet sdk if necessary
         Dictionary<GitFileMetadataName, string> globalJsonMetadata = null;
@@ -666,7 +666,7 @@ private static bool IsMaestroManagedFeed(string feed)
                FeedConstants.AzureStorageProxyFeedPattern.IsMatch(feed);
     }
 
-    public XmlDocument UpdatePackageSources(XmlDocument nugetConfig, Dictionary<string, HashSet<string>> maestroManagedFeedsByRepo)
+    public XmlDocument UpdatePackageSources(XmlDocument nugetConfig, Dictionary<string, HashSet<string>> maestroManagedFeedsByRepo, Dictionary<string, HashSet<string>> packageToFeedMapping = null)
     {
         // Reconstruct the PackageSources section with the feeds
         XmlNode packageSourcesNode = nugetConfig.SelectSingleNode("//configuration/packageSources");
@@ -734,6 +734,8 @@ public XmlDocument UpdatePackageSources(XmlDocument nugetConfig, Dictionary<stri
 
         CreateOrUpdateDisabledSourcesBlock(nugetConfig, maestroManagedFeedsByRepo, FeedConstants.MaestroManagedInternalFeedPrefix);
 
+        CreateOrUpdatePackageSourceMappingBlock(nugetConfig, maestroManagedFeedsByRepo, packageToFeedMapping);
+
         return nugetConfig;
     }
 
@@ -953,6 +955,151 @@ private void InsertManagedPackagesBlock(XmlDocument nugetConfig, XmlNode package
         }
     }
 
+    /// <summary>
+    /// Create or update the packageSourceMapping section to map packages to their corresponding darc feeds
+    /// </summary>
+    /// <param name="nugetConfig">The NuGet.config XML document</param>
+    /// <param name="maestroManagedFeedsByRepo">Managed feeds organized by repository</param>
+    /// <param name="packageToFeedMapping">Mapping of package names to their feed locations</param>
+    private void CreateOrUpdatePackageSourceMappingBlock(XmlDocument nugetConfig, Dictionary<string, HashSet<string>> maestroManagedFeedsByRepo, Dictionary<string, HashSet<string>> packageToFeedMapping)
+    {
+        if (packageToFeedMapping == null || packageToFeedMapping.Count == 0)
+        {
+            return;
+        }
+
+        // Get or create the packageSourceMapping node
+        XmlNode packageSourceMappingNode = nugetConfig.SelectSingleNode("//configuration/packageSourceMapping");
+        if (packageSourceMappingNode == null)
+        {
+            XmlNode configNode = nugetConfig.SelectSingleNode("//configuration");
+            if (configNode == null)
+            {
+                return;
+            }
+
+            packageSourceMappingNode = nugetConfig.CreateElement("packageSourceMapping");
+            configNode.AppendChild(packageSourceMappingNode);
+        }
+
+        // Create a reverse mapping from feed URL to darc key name
+        var feedUrlToKeyMapping = new Dictionary<string, string>();
+        foreach (var repoFeeds in maestroManagedFeedsByRepo.Values)
+        {
+            var managedSources = GetManagedPackageSources(repoFeeds);
+            foreach (var (key, feed) in managedSources)
+            {
+                feedUrlToKeyMapping[feed] = key;
+            }
+        }
+
+        // Remove existing managed packageSource mappings (those managed by Maestro)
+        var nodesToRemove = new List<XmlNode>();
+        XmlNode currentNode = packageSourceMappingNode.FirstChild;
+        bool withinMaestroComments = false;
+
+        while (currentNode != null)
+        {
+            if (currentNode.NodeType == XmlNodeType.Comment)
+            {
+                if (currentNode.Value.Equals(MaestroBeginComment, StringComparison.OrdinalIgnoreCase))
+                {
+                    withinMaestroComments = true;
+                    nodesToRemove.Add(currentNode);
+                }
+                else if (currentNode.Value.Equals(MaestroEndComment, StringComparison.OrdinalIgnoreCase))
+                {
+                    withinMaestroComments = false;
+                    nodesToRemove.Add(currentNode);
+                }
+                else if (withinMaestroComments)
+                {
+                    nodesToRemove.Add(currentNode);
+                }
+            }
+            else if (currentNode.NodeType == XmlNodeType.Element && withinMaestroComments)
+            {
+                if (currentNode.Name.Equals("packageSource", StringComparison.OrdinalIgnoreCase))
+                {
+                    var keyAttr = currentNode.Attributes?["key"]?.Value;
+                    if (!string.IsNullOrEmpty(keyAttr) && keyAttr.StartsWith("darc-", StringComparison.OrdinalIgnoreCase))
+                    {
+                        nodesToRemove.Add(currentNode);
+                    }
+                }
+            }
+
+            currentNode = currentNode.NextSibling;
+        }
+
+        // Remove the collected nodes
+        foreach (var node in nodesToRemove)
+        {
+            node.ParentNode?.RemoveChild(node);
+        }
+
+        // Build mapping from darc feed keys to packages that should use them
+        var feedKeyToPackages = new Dictionary<string, HashSet<string>>();
+        foreach (var (packageName, feedUrls) in packageToFeedMapping)
+        {
+            // Only process packages that are in Maestro-managed feeds
+            var maestroManagedFeeds = feedUrls.Where(feedUrl => IsMaestroManagedFeed(feedUrl)).ToList();
+
+            foreach (var feedUrl in maestroManagedFeeds)
+            {
+                if (feedUrlToKeyMapping.TryGetValue(feedUrl, out string feedKey))
+                {
+                    if (!feedKeyToPackages.ContainsKey(feedKey))
+                    {
+                        feedKeyToPackages[feedKey] = [];
+                    }
+                    feedKeyToPackages[feedKey].Add(packageName);
+                }
+            }
+        }
+
+        // Only add mappings if we have darc feeds with packages
+        if (feedKeyToPackages.Count > 0)
+        {
+            // Find insertion point (after any existing clear element)
+            XmlNode insertAfterNode = null;
+            currentNode = packageSourceMappingNode.FirstChild;
+            while (currentNode != null)
+            {
+                if (currentNode.Name.Equals("clear", StringComparison.OrdinalIgnoreCase))
+                {
+                    insertAfterNode = currentNode;
+                    break;
+                }
+                currentNode = currentNode.NextSibling;
+            }
+
+            // Add begin comment
+            var beginComment = nugetConfig.CreateComment(MaestroBeginComment);
+            insertAfterNode = packageSourceMappingNode.InsertAfter(beginComment, insertAfterNode);
+
+            // Add packageSource entries for each darc feed with its packages
+            foreach (var (feedKey, packages) in feedKeyToPackages.OrderBy(kvp => kvp.Key))
+            {
+                var packageSourceElement = nugetConfig.CreateElement("packageSource");
+                packageSourceElement.SetAttribute("key", feedKey);
+
+                foreach (var packageName in packages.OrderBy(p => p))
+                {
+                    var packageElement = nugetConfig.CreateElement("package");
+                    packageElement.SetAttribute("pattern", packageName);
+                    packageSourceElement.AppendChild(packageElement);
+                }
+
+                insertAfterNode = packageSourceMappingNode.InsertAfter(packageSourceElement, insertAfterNode);
+            }
+
+            // Add end comment
+            var endComment = nugetConfig.CreateComment(MaestroEndComment);
+            packageSourceMappingNode.InsertAfter(endComment, insertAfterNode);
+        }
+    }
+
     private static XmlComment GetFirstMatchingComment(XmlNode nodeToCheck, string commentText)
     {
         if (nodeToCheck.HasChildNodes)

src/Microsoft.DotNet.Darc/DarcLib/Helpers/IDependencyFileManager.cs

@@ -82,7 +82,7 @@ Task<GitFileContentContainer> UpdateDependencyFiles(
         bool? repoHasVersionDetailsProps = null,
         UnixPath? relativeBasePath = null);
 
-    XmlDocument UpdatePackageSources(XmlDocument nugetConfig, Dictionary<string, HashSet<string>> maestroManagedFeedsByRepo);
+    XmlDocument UpdatePackageSources(XmlDocument nugetConfig, Dictionary<string, HashSet<string>> maestroManagedFeedsByRepo, Dictionary<string, HashSet<string>>? packageToFeedMapping = null);
 
     Task<bool> Verify(string repo, string branch);
 

test/Microsoft.DotNet.Darc.Tests/DependencyFileManagerTests.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
 using System.Xml;
 using FluentAssertions;
@@ -114,6 +115,67 @@ public async Task UpdatePackageSourcesTests(string testName, string[] managedFee
         doubleUpdatedfile.Content.Should().Be(expectedOutputText, "Repeated invocation of UpdatePackageSources() caused incremental changes to nuget.config");
     }
 
+    [Test]
+    public async Task UpdatePackageSourcesWithPackageMappingsTest()
+    {
+        var dependencyFileManager = new DependencyFileManager((IGitRepo)null, new VersionDetailsParser(), NullLogger.Instance);
+
+        string testName = "AddPackageSourceMappingsForDarcFeeds";
+        string[] managedFeeds = new string[] {
+            "https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-standard-a5b5f2e1/nuget/v3/index.json",
+            "https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-corefx-4ac4c036/nuget/v3/index.json"
+        };
+
+        string inputNugetPath = Path.Combine(
+            Environment.CurrentDirectory,
+            TestInputsRootDir,
+            ConfigFilesInput,
+            testName,
+            InputNugetConfigFile);
+        string inputXmlContent = await File.ReadAllTextAsync(inputNugetPath);
+        var inputNuGetConfigFile = DependencyFileManager.GetXmlDocument(inputXmlContent);
+
+        var configFileUpdateData = new Dictionary<string, HashSet<string>>
+        {
+            { "testKey", new HashSet<string>(managedFeeds) }
+        };
+        var managedFeedsForTest = dependencyFileManager.FlattenLocationsAndSplitIntoGroups(configFileUpdateData);
+
+        // Create package-to-feed mapping for this test
+        var packageToFeedMapping = new Dictionary<string, HashSet<string>>
+        {
+            { "System.Collections.Immutable", new HashSet<string> { "https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-corefx-4ac4c036/nuget/v3/index.json" } },
+            { "System.Text.Json", new HashSet<string> { "https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-corefx-4ac4c036/nuget/v3/index.json" } },
+            { "NETStandard.Library", new HashSet<string> { "https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-standard-a5b5f2e1/nuget/v3/index.json" } }
+        };
+
+        XmlDocument updatedConfigFile =
+            dependencyFileManager.UpdatePackageSources(inputNuGetConfigFile, managedFeedsForTest, packageToFeedMapping);
+
+        var outputNugetPath = Path.Combine(
+            Environment.CurrentDirectory,
+            TestInputsRootDir,
+            ConfigFilesInput,
+            testName,
+            OutputNugetConfigFile);
+        string expectedOutputText = await File.ReadAllTextAsync(outputNugetPath);
+
+        // Dump the output xml using the git file manager
+        var file = new GitFile(null, updatedConfigFile);
+
+        // Normalize the \r\n newlines in the expected output to \n if they
+        // exist (GitFile normalizes these before writing)
+        expectedOutputText = expectedOutputText.Replace(Environment.NewLine, "\n");
+
+        // Use NUnit's Assert instead of FluentAssertions due to whitespace sensitivity
+        NUnit.Framework.Assert.AreEqual(expectedOutputText, file.Content);
+
+        // Test idempotency
+        XmlDocument doubleUpdatedConfigFile = dependencyFileManager.UpdatePackageSources(updatedConfigFile, managedFeedsForTest, packageToFeedMapping);
+        var doubleUpdatedfile = new GitFile(null, doubleUpdatedConfigFile);
+        NUnit.Framework.Assert.AreEqual(expectedOutputText, doubleUpdatedfile.Content, "Repeated invocation of UpdatePackageSources() caused incremental changes to nuget.config");
+    }
+
     [TestCase("SimpleDuplicated.props", true)]
     [TestCase("DuplicatedSameConditions.props", true)]
     [TestCase("AlternateNamesDuplicated.props", true)]

test/Microsoft.DotNet.Darc.Tests/inputs/NugetConfigFiles/AddPackageSourceMappingsForDarcFeeds/NuGet.expected-no-mappings.config

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- This test case exercises adding packageSourceMappings for darc feeds when a config already has packageSourceMappings
+     Expected behaviors:
+       - packageSourceMappings section should be updated with darc feed mappings
+       - Existing packageSourceMappings should be preserved
+       - Darc package mappings should be added within Maestro managed comments
+ -->
+<configuration>
+  <packageSources>
+    <clear />
+    <!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->
+    <!--  Begin: Package sources from dotnet-corefx -->
+    <add key="darc-pub-dotnet-corefx-4ac4c03" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-corefx-4ac4c036/nuget/v3/index.json" />
+    <!--  End: Package sources from dotnet-corefx -->
+    <!--  Begin: Package sources from dotnet-standard -->
+    <add key="darc-pub-dotnet-standard-a5b5f2e" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-standard-a5b5f2e1/nuget/v3/index.json" />
+    <!--  End: Package sources from dotnet-standard -->
+    <!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
+    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <packageSourceMapping>
+    <clear />
+    <packageSource key="nuget.org">
+      <package pattern="Newtonsoft.Json" />
+      <package pattern="System.*" />
+    </packageSource>
+    <packageSource key="dotnet-core">
+      <package pattern="Microsoft.*" />
+    </packageSource>
+  </packageSourceMapping>
+  <disabledPackageSources />
+</configuration>

test/Microsoft.DotNet.Darc.Tests/inputs/NugetConfigFiles/AddPackageSourceMappingsForDarcFeeds/NuGet.input.config

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- This test case exercises adding packageSourceMappings for darc feeds when a config already has packageSourceMappings
+     Expected behaviors:
+       - packageSourceMappings section should be updated with darc feed mappings
+       - Existing packageSourceMappings should be preserved
+       - Darc package mappings should be added within Maestro managed comments
+ -->
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <packageSourceMapping>
+    <clear />
+    <packageSource key="nuget.org">
+      <package pattern="Newtonsoft.Json" />
+      <package pattern="System.*" />
+    </packageSource>
+    <packageSource key="dotnet-core">
+      <package pattern="Microsoft.*" />
+    </packageSource>
+  </packageSourceMapping>
+  <disabledPackageSources />
+</configuration>

test/Microsoft.DotNet.Darc.Tests/inputs/NugetConfigFiles/AddPackageSourceMappingsForDarcFeeds/NuGet.output.config

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- This test case exercises adding packageSourceMappings for darc feeds when a config already has packageSourceMappings
+     Expected behaviors:
+       - packageSourceMappings section should be updated with darc feed mappings
+       - Existing packageSourceMappings should be preserved
+       - Darc package mappings should be added within Maestro managed comments
+ -->
+<configuration>
+  <packageSources>
+    <clear />
+    <!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->
+    <!--  Begin: Package sources from dotnet-corefx -->
+    <add key="darc-pub-dotnet-corefx-4ac4c03" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-corefx-4ac4c036/nuget/v3/index.json" />
+    <!--  End: Package sources from dotnet-corefx -->
+    <!--  Begin: Package sources from dotnet-standard -->
+    <add key="darc-pub-dotnet-standard-a5b5f2e" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-standard-a5b5f2e1/nuget/v3/index.json" />
+    <!--  End: Package sources from dotnet-standard -->
+    <!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
+    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <packageSourceMapping>
+    <clear />
+    <!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->
+    <packageSource key="darc-pub-dotnet-corefx-4ac4c03">
+      <package pattern="System.Collections.Immutable" />
+      <package pattern="System.Text.Json" />
+    </packageSource>
+    <packageSource key="darc-pub-dotnet-standard-a5b5f2e">
+      <package pattern="NETStandard.Library" />
+    </packageSource>
+    <!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
+    <packageSource key="nuget.org">
+      <package pattern="Newtonsoft.Json" />
+      <package pattern="System.*" />
+    </packageSource>
+    <packageSource key="dotnet-core">
+      <package pattern="Microsoft.*" />
+    </packageSource>
+  </packageSourceMapping>
+  <disabledPackageSources />
+</configuration>