Introduced codeowner-extractor. (#3719)

* Introduced codeowner-extractor.

* Refactor using implicit principal and vault based config (#3751)

* Refactor using implicit principal and vault based config
* Cleanup logging and skipping logic in Processor. Use bulk github user mapping api.
* Avoid using arrays with IConfiguration
* Improve logging in GitHubToAADConverter and pipeline owner extractor

Co-authored-by: Nicholi Harris <t-nharris@microsoft.com>
Co-authored-by: Patrick Hallisey <pahallis@microsoft.com>

tools/identity-resolution/Helpers/GitHubToAADConverter.cs

@@ -1,8 +1,11 @@
 using System;
+using System.Net;
 using System.Net.Http;
+using System.Threading.Tasks;
 using Azure.Core;
 using Azure.Identity;
 using Microsoft.Extensions.Logging;
+using Models.OpenSourcePortal;
 using Newtonsoft.Json;
 
 namespace Azure.Sdk.Tools.NotificationConfiguration.Helpers
@@ -48,18 +51,46 @@ public GitHubToAADConverter(
         /// <param name="githubUserName">github user name</param>
         /// <returns>Aad user principal name</returns>
         public string GetUserPrincipalNameFromGithub(string githubUserName)
+        {
+            return GetUserPrincipalNameFromGithubAsync(githubUserName).Result;
+        }
+
+        public async Task<string> GetUserPrincipalNameFromGithubAsync(string githubUserName)
         {
             try
             {
-                var responseJsonString = client.GetStringAsync($"https://repos.opensource.microsoft.com/api/people/links/github/{githubUserName}").Result;
+                var responseJsonString = await client.GetStringAsync($"https://repos.opensource.microsoft.com/api/people/links/github/{githubUserName}");
                 dynamic contentJson = JsonConvert.DeserializeObject(responseJsonString);
                 return contentJson.aad.userPrincipalName;
             }
+            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                logger.LogWarning("Github username {Username} not found", githubUserName);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex.Message);
+            }
+
+            return null;
+        }
+
+        public async Task<UserLink[]> GetPeopleLinksAsync()
+        {
+            try
+            {
+                logger.LogInformation("Calling GET https://repos.opensource.microsoft.com/api/people/links");
+                var responseJsonString = await client.GetStringAsync($"https://repos.opensource.microsoft.com/api/people/links");
+                var allLinks = JsonConvert.DeserializeObject<UserLink[]>(responseJsonString);
+
+                return allLinks;
+            }
             catch (Exception ex)
             {
                 logger.LogError(ex.Message);
-                return null;
             }
+
+            return null;
         }
     }
 }

tools/identity-resolution/Models/OpenSourcePortal/AadUserDetail.cs

@@ -0,0 +1,15 @@
+namespace Models.OpenSourcePortal
+{
+    public class AadUserDetail
+    {
+        public string Alias { get; set; }
+
+        public string PreferredName { get; set; }
+
+        public string UserPrincipalName { get; set; }
+
+        public string Id { get; set; }
+
+        public string EmailAddress { get; set; }
+    }
+}

tools/identity-resolution/Models/OpenSourcePortal/GitHubUserDetail.cs

@@ -0,0 +1,13 @@
+namespace Models.OpenSourcePortal
+{
+    public class GitHubUserDetail
+    {
+        public int Id { get; set; }
+
+        public string Login { get; set; }
+
+        public string[] Organizations { get; set; }
+
+        public string Avatar { get; set; }
+    }
+}

tools/identity-resolution/Models/OpenSourcePortal/UserLink.cs

@@ -0,0 +1,13 @@
+namespace Models.OpenSourcePortal
+{
+    public class UserLink
+    {
+        public GitHubUserDetail GitHub { get; set; }
+
+        public AadUserDetail Aad { get; set; }
+
+        public bool IsServiceAccount { get; set; }
+
+        public string ServiceAccountContact { get; set; }
+    }
+}

tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Azure.Sdk.Tools.PipelineOwnersExtractor.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <PackAsTool>true</PackAsTool>
+    <ToolCommandName>pipeline-owners-extractor</ToolCommandName>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Azure.Security.KeyVault.Secrets" Version="4.3.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\identity-resolution\identity-resolution.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Content Include="appsettings.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
+</Project>

tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/ISecretClientProvider.cs

@@ -0,0 +1,11 @@
+using System;
+
+using Azure.Security.KeyVault.Secrets;
+
+namespace Azure.Sdk.Tools.PipelineOwnersExtractor.Configuration
+{
+    public interface ISecretClientProvider
+    {
+        SecretClient GetSecretClient(Uri vaultUri);
+    }
+}

tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/PipelineOwnerSettings.cs

@@ -0,0 +1,19 @@
+namespace Azure.Sdk.Tools.PipelineOwnersExtractor.Configuration
+{
+    public class PipelineOwnerSettings
+    {
+        public string Account { get; set; }
+
+        public string Projects { get; set; }
+
+        public string OpenSourceAadAppId { get; set; }
+
+        public string OpenSourceAadSecret { get; set; }
+
+        public string OpenSourceAadTenantId { get; set; }
+
+        public string AzureDevOpsPat { get; set; }
+
+        public string Output { get; set; }
+    }
+}

tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/PostConfigureKeyVaultSettings.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Linq;
+using System.Text.RegularExpressions;
+
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Azure.Sdk.Tools.PipelineOwnersExtractor.Configuration
+{
+    public class PostConfigureKeyVaultSettings<T> : IPostConfigureOptions<T> where T : class
+    {
+        private static readonly Regex secretRegex = new Regex(@"(?<vault>https://.*?\.vault\.azure\.net)/secrets/(?<secret>.*)", RegexOptions.Compiled, TimeSpan.FromSeconds(5));
+        private readonly ILogger logger;
+        private readonly ISecretClientProvider secretClientProvider;
+
+        public PostConfigureKeyVaultSettings(ILogger<PostConfigureKeyVaultSettings<T>> logger, ISecretClientProvider secretClientProvider)
+        {
+            this.logger = logger;
+            this.secretClientProvider = secretClientProvider;
+        }
+
+        public void PostConfigure(string name, T options)
+        {
+            var stringProperties = typeof(T)
+                .GetProperties()
+                .Where(x => x.PropertyType == typeof(string));
+
+            foreach (var property in stringProperties)
+            {
+                var value = (string)property.GetValue(options);
+
+                if (value != null)
+                {
+                    var match = secretRegex.Match(value);
+
+                    if (match.Success)
+                    {
+                        var vaultUrl = match.Groups["vault"].Value;
+                        var secretName = match.Groups["secret"].Value;
+
+                        try
+                        {
+                            var secretClient = this.secretClientProvider.GetSecretClient(new Uri(vaultUrl));
+                            this.logger.LogInformation("Replacing setting property {PropertyName} with value from secret {SecretUrl}", property.Name, value);
+
+                            var response = secretClient.GetSecret(secretName);
+                            var secret = response.Value;
+
+                            property.SetValue(options, secret.Value);
+                        }
+                        catch (Exception exception)
+                        {
+                            this.logger.LogError(exception, "Unable to read secret {SecretName} from vault {VaultUrl}", secretName, vaultUrl);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/SecretClientProvider.cs

@@ -0,0 +1,22 @@
+using System;
+
+using Azure.Core;
+using Azure.Security.KeyVault.Secrets;
+
+namespace Azure.Sdk.Tools.PipelineOwnersExtractor.Configuration
+{
+    public class SecretClientProvider : ISecretClientProvider
+    {
+        private readonly TokenCredential tokenCredential;
+
+        public SecretClientProvider(TokenCredential tokenCredential)
+        {
+            this.tokenCredential = tokenCredential;
+        }
+
+        public SecretClient GetSecretClient(Uri vaultUri)
+        {
+            return new SecretClient(vaultUri, this.tokenCredential);
+        }
+    }
+}

tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Extensions.cs

@@ -0,0 +1,48 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace Azure.Sdk.Tools.PipelineOwnersExtractor
+{
+    public static class Extensions
+    {
+        public static async Task<T[]> LimitConcurrencyAsync<T>(this IEnumerable<Task<T>> tasks, int concurrencyLimit = 1)
+        {
+            if (concurrencyLimit == int.MaxValue) 
+            { 
+                return await Task.WhenAll(tasks);
+            }
+
+            var results = new List<T>();
+
+            if (concurrencyLimit == 1)
+            {
+                foreach (var task in tasks)
+                {
+                    results.Add(await task);
+                }
+
+                return results.ToArray();
+            }
+
+            var pending = new List<Task<T>>();
+
+            foreach (var task in tasks)
+            {
+                pending.Add(task);
+
+                if (pending.Count < concurrencyLimit)
+                {
+                    continue; 
+                }
+
+                var completed = await Task.WhenAny(pending);
+                pending.Remove(completed);
+                results.Add(await completed);
+            }
+
+            results.AddRange(await Task.WhenAll(pending));
+
+            return results.ToArray();
+        }
+    }
+}

tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/GitHubService.cs

@@ -0,0 +1,74 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Azure.Sdk.Tools.CodeOwnersParser;
+using Microsoft.Extensions.Logging;
+
+namespace Azure.Sdk.Tools.PipelineOwnersExtractor
+{
+    /// <summary>
+    /// Interface for interacting with GitHub
+    /// </summary>
+    public class GitHubService
+    {
+        private static readonly HttpClient httpClient = new HttpClient();
+
+        private readonly ILogger<GitHubService> logger;
+        private readonly ConcurrentDictionary<string, List<CodeOwnerEntry>> codeOwnersFileCache;
+
+        /// <summary>
+        /// Creates a new GitHubService
+        /// </summary>
+        /// <param name="logger">Logger</param>
+        public GitHubService(ILogger<GitHubService> logger)
+        {
+            this.logger = logger;
+            this.codeOwnersFileCache = new ConcurrentDictionary<string, List<CodeOwnerEntry>>();
+        }
+
+        /// <summary>
+        /// Looks for CODEOWNERS in the main branch of the given repo URL using cache
+        /// </summary>
+        /// <param name="repoUrl">GitHub repository URL</param>
+        /// <returns>Contents fo the located CODEOWNERS file</returns>
+        public async Task<List<CodeOwnerEntry>> GetCodeOwnersFile(Uri repoUrl)
+        {
+            List<CodeOwnerEntry> result;
+            if (codeOwnersFileCache.TryGetValue(repoUrl.ToString(), out result))
+            {
+                return result;
+            }
+
+            result = await GetCodeownersFileImpl(repoUrl);
+            codeOwnersFileCache.TryAdd(repoUrl.ToString(), result);
+            return result;
+        }
+
+        /// <summary>
+        /// Looks for CODEOWNERS in the main branch of the given repo URL
+        /// </summary>
+        /// <param name="repoUrl"></param>
+        /// <returns></returns>
+        private async Task<List<CodeOwnerEntry>> GetCodeownersFileImpl(Uri repoUrl)
+        {
+            // Gets the repo path from the URL
+            var relevantPathParts = repoUrl.Segments.Skip(1).Take(2);
+            var repoPath = string.Join("", relevantPathParts);
+
+            var codeOwnersUrl = $"https://raw.githubusercontent.com/{repoPath}/main/.github/CODEOWNERS";
+            var result = await httpClient.GetAsync(codeOwnersUrl);
+            if (result.IsSuccessStatusCode)
+            {
+                this.logger.LogInformation("Retrieved CODEOWNERS file URL = {0}", codeOwnersUrl);
+                return CodeOwnersFile.ParseContent(await result.Content.ReadAsStringAsync());
+            }
+
+            this.logger.LogWarning("Could not retrieve CODEOWNERS file URL = {0} ResponseCode = {1}", codeOwnersUrl, result.StatusCode);
+            return default;
+        }
+
+    }
+}