diff --git a/tools/identity-resolution/Helpers/GitHubToAADConverter.cs b/tools/identity-resolution/Helpers/GitHubToAADConverter.cs index 49d3067acc9..f3235ec6ea3 100644 --- a/tools/identity-resolution/Helpers/GitHubToAADConverter.cs +++ b/tools/identity-resolution/Helpers/GitHubToAADConverter.cs @@ -5,6 +5,7 @@ using Azure.Core; using Azure.Identity; using Microsoft.Extensions.Logging; +using Models.OpenSourcePortal; using Newtonsoft.Json; namespace Azure.Sdk.Tools.NotificationConfiguration.Helpers @@ -51,7 +52,7 @@ public GitHubToAADConverter( /// Aad user principal name public string GetUserPrincipalNameFromGithub(string githubUserName) { - return GetUserPrincipalNameFromGithubAsync(githubUserName).Result; + return GetUserPrincipalNameFromGithubAsync(githubUserName).Result; } public async Task GetUserPrincipalNameFromGithubAsync(string githubUserName) @@ -73,5 +74,23 @@ public async Task GetUserPrincipalNameFromGithubAsync(string githubUserN return null; } + + public async Task GetPeopleLinksAsync() + { + try + { + logger.LogInformation("Calling GET https://repos.opensource.microsoft.com/api/people/links"); + var responseJsonString = await client.GetStringAsync($"https://repos.opensource.microsoft.com/api/people/links"); + var allLinks = JsonConvert.DeserializeObject(responseJsonString); + + return allLinks; + } + catch (Exception ex) + { + logger.LogError(ex.Message); + } + + return null; + } } } diff --git a/tools/identity-resolution/Models/OpenSourcePortal/AadUserDetail.cs b/tools/identity-resolution/Models/OpenSourcePortal/AadUserDetail.cs new file mode 100644 index 00000000000..f43f2cf2456 --- /dev/null +++ b/tools/identity-resolution/Models/OpenSourcePortal/AadUserDetail.cs @@ -0,0 +1,15 @@ +namespace Models.OpenSourcePortal +{ + public class AadUserDetail + { + public string Alias { get; set; } + + public string PreferredName { get; set; } + + public string UserPrincipalName { get; set; } + + public string Id { get; set; } + + public string EmailAddress { get; set; } + } +} diff --git a/tools/identity-resolution/Models/OpenSourcePortal/GitHubUserDetail.cs b/tools/identity-resolution/Models/OpenSourcePortal/GitHubUserDetail.cs new file mode 100644 index 00000000000..246f88e192d --- /dev/null +++ b/tools/identity-resolution/Models/OpenSourcePortal/GitHubUserDetail.cs @@ -0,0 +1,13 @@ +namespace Models.OpenSourcePortal +{ + public class GitHubUserDetail + { + public int Id { get; set; } + + public string Login { get; set; } + + public string[] Organizations { get; set; } + + public string Avatar { get; set; } + } +} diff --git a/tools/identity-resolution/Models/OpenSourcePortal/UserLink.cs b/tools/identity-resolution/Models/OpenSourcePortal/UserLink.cs new file mode 100644 index 00000000000..26ed041bb91 --- /dev/null +++ b/tools/identity-resolution/Models/OpenSourcePortal/UserLink.cs @@ -0,0 +1,13 @@ +namespace Models.OpenSourcePortal +{ + public class UserLink + { + public GitHubUserDetail GitHub { get; set; } + + public AadUserDetail Aad { get; set; } + + public bool IsServiceAccount { get; set; } + + public string ServiceAccountContact { get; set; } + } +} diff --git a/tools/pipeline-codeowners-extractor/Azure.Sdk.Tools.PipelineCodeownerExtractor.csproj b/tools/pipeline-codeowners-extractor/Azure.Sdk.Tools.PipelineCodeownerExtractor.csproj deleted file mode 100644 index d663c6c9f09..00000000000 --- a/tools/pipeline-codeowners-extractor/Azure.Sdk.Tools.PipelineCodeownerExtractor.csproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - Exe - net5.0 - true - pipeline-codeowner-extractor - - - - - - - - - - - - diff --git a/tools/pipeline-codeowners-extractor/Extensions.cs b/tools/pipeline-codeowners-extractor/Extensions.cs deleted file mode 100644 index e65e5165f9b..00000000000 --- a/tools/pipeline-codeowners-extractor/Extensions.cs +++ /dev/null @@ -1,96 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineCodeownerExtractor -{ - public static class Extensions - { - public static Task WhenAll(this IEnumerable> tasks) - { - return Task.WhenAll(tasks); - } - - public static Task WhenAll(this IEnumerable tasks) - { - return Task.WhenAll(tasks); - } - - - public static async Task LimitConcurrencyAsync(this IEnumerable tasks, int concurrencyLimit = 1) - { - if (concurrencyLimit == int.MaxValue) - { - await Task.WhenAll(tasks); - return; - } - - if (concurrencyLimit == 1) - { - foreach (var task in tasks) - { - await task; - } - - return; - } - - var pending = new List(); - - foreach (var task in tasks) - { - pending.Add(task); - - if (pending.Count < concurrencyLimit) - { - continue; - } - - var completed = await Task.WhenAny(pending); - - pending.Remove(completed); - } - - await Task.WhenAll(pending); - } - - public static async Task LimitConcurrencyAsync(this IEnumerable> tasks, int concurrencyLimit = 1) - { - if (concurrencyLimit == int.MaxValue) - { - return await Task.WhenAll(tasks); - } - - var results = new List(); - - if (concurrencyLimit == 1) - { - foreach (var task in tasks) - { - results.Add(await task); - } - - return results.ToArray(); - } - - var pending = new List>(); - - foreach (var task in tasks) - { - pending.Add(task); - - if (pending.Count < concurrencyLimit) - { - continue; - } - - var completed = await Task.WhenAny(pending); - pending.Remove(completed); - results.Add(await completed); - } - - results.AddRange(await Task.WhenAll(pending)); - - return results.ToArray(); - } - } -} diff --git a/tools/pipeline-codeowners-extractor/Program.cs b/tools/pipeline-codeowners-extractor/Program.cs deleted file mode 100644 index e7ccffddd4c..00000000000 --- a/tools/pipeline-codeowners-extractor/Program.cs +++ /dev/null @@ -1,172 +0,0 @@ -using Microsoft.Extensions.Logging; -using Microsoft.TeamFoundation.Build.WebApi; -using Azure.Sdk.Tools.NotificationConfiguration.Enums; -using Azure.Sdk.Tools.NotificationConfiguration.Helpers; -using Azure.Sdk.Tools.NotificationConfiguration.Models; -using Azure.Sdk.Tools.NotificationConfiguration.Services; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Azure.Sdk.Tools.CodeOwnersParser; -using Azure.Identity; -using System.IO; -using Newtonsoft.Json; - -namespace Azure.Sdk.Tools.PipelineCodeownerExtractor -{ - class Program - { - // Type 2 maps to a pipeline YAML file in the repository - private const int PipelineYamlProcessType = 2; - private static readonly Dictionary githubPrincipalNameCache = new Dictionary(); - private static ILogger logger; - private static GitHubToAADConverter githubToAadResolver; - private static GitHubService gitHubService; - - /// - /// Synchronizes CODEOWNERS contacts to appropriate DevOps groups - /// - /// Azure DevOps organization name - /// Azure DevOps project name - /// Personal Access Token environment variable name - /// AAD App ID environment variable name (OpensourceAPI access) - /// AAD App Secret environment variable name (OpensourceAPI access) - /// AAD Tenant environment variable name (OpensourceAPI access) - /// Azure DevOps path prefix (e.g. "\net") - /// Output file path - /// - public static async Task Main( - string organization, - string[] projects, - string aadAppIdVar = "", - string aadAppSecretVar = "", - string aadTenantVar = "", - string devOpsTokenVar = "", - string pathPrefix = "", - string output = "pipeline-owners.json" - ) - { - using var loggerFactory = LoggerFactory.Create(builder => builder.AddSimpleConsole(options => options.SingleLine = true)); - - var devOpsService = AzureDevOpsService.CreateAzureDevOpsService( - Environment.GetEnvironmentVariable(devOpsTokenVar), - $"https://dev.azure.com/{organization}/", - loggerFactory.CreateLogger() - ); - - gitHubService = new GitHubService(loggerFactory.CreateLogger()); - - var credential = new ClientSecretCredential( - Environment.GetEnvironmentVariable(aadTenantVar), - Environment.GetEnvironmentVariable(aadAppIdVar), - Environment.GetEnvironmentVariable(aadAppSecretVar)); - - githubToAadResolver = new GitHubToAADConverter( - credential, - loggerFactory.CreateLogger() - ); - - logger = loggerFactory.CreateLogger(string.Empty); - - var pipelineResults = await Task.WhenAll(projects.Select(x => devOpsService.GetPipelinesAsync(x))); - - var pipelines = pipelineResults.SelectMany(x => x); - - var filteredPipelines = pipelines - .Where(pipeline => pipeline.Path.StartsWith(pathPrefix)) - .ToArray(); - - var repositoryUrls = GetDistinctRepositoryUrls(filteredPipelines); - - var codeOwnerEntriesByRepository = await GetCodeOwnerEntriesAsync(repositoryUrls); - - var pipelineOwners = await AssociateOwnersToPipelinesAsync(filteredPipelines, codeOwnerEntriesByRepository); - - var outputContent = pipelineOwners.ToDictionary(x => x.Pipeline.Id, x => x.Owners); - - File.WriteAllText(output, JsonConvert.SerializeObject(outputContent, Formatting.Indented)); - } - - private static async Task<(BuildDefinition Pipeline, string[] Owners)[]> AssociateOwnersToPipelinesAsync(BuildDefinition[] filteredPipelines, Dictionary> codeOwnerEntriesByRepository) - { - var githubPipelineOwners = new List<(BuildDefinition Pipeline, string[] Owners)>(); - - foreach (var pipeline in filteredPipelines) - { - logger.LogInformation("Pipeline Name = {0}", pipeline.Name); - - if (pipeline.Process.Type != PipelineYamlProcessType || !(pipeline.Process is YamlProcess process)) - { - continue; - } - - if (!pipeline.Repository.Properties.TryGetValue("manageUrl", out var managementUrl)) - { - logger.LogInformation("ManagementURL not found, skipping sync"); - continue; - } - - if (!codeOwnerEntriesByRepository.TryGetValue(managementUrl, out var codeOwnerEntries)) - { - logger.LogInformation("CODEOWNERS file not found, skipping sync"); - continue; - } - - logger.LogInformation("Searching CODEOWNERS for matching path for {0}", process.YamlFilename); - var codeOwnerEntry = CodeOwnersFile.FindOwnersForClosestMatch(codeOwnerEntries, process.YamlFilename); - codeOwnerEntry.FilterOutNonUserAliases(); - - logger.LogInformation("Matching Contacts Path = {0}, NumContacts = {1}", process.YamlFilename, codeOwnerEntry.Owners.Count); - - // Get set of team members in the CODEOWNERS file - var codeownerPrincipals = codeOwnerEntry.Owners.ToArray(); - - githubPipelineOwners.Add((pipeline, codeownerPrincipals)); - } - - var distinctGithubAliases = githubPipelineOwners.SelectMany(x => x.Owners).Distinct().ToArray(); - - var microsoftAliasMap = await distinctGithubAliases - .Select(async githubAlias => new - { - Github = githubAlias, - Microsoft = await githubToAadResolver.GetUserPrincipalNameFromGithubAsync(githubAlias), - }) - .LimitConcurrencyAsync(10) - .ContinueWith(x => x.Result.ToDictionary(x => x.Github, x => x.Microsoft)); - - var microsoftPipelineOwners = githubPipelineOwners - .Select(x => ( - x.Pipeline, - x.Owners - .Select(o => microsoftAliasMap[o]) - .Where(o => o != null) - .ToArray())) - .ToArray(); - - return microsoftPipelineOwners; - } - - private static async Task>> GetCodeOwnerEntriesAsync(string[] repositoryUrls) - { - var tasks = repositoryUrls.Select(async url => new - { - RepositoryUrl = url, - CodeOwners = await gitHubService.GetCodeownersFile(new Uri(url)) - }); - - var taskResults = await Task.WhenAll(tasks); - - return taskResults.Where(x => x.CodeOwners != null).ToDictionary(x => x.RepositoryUrl, x => x.CodeOwners); - } - - private static string[] GetDistinctRepositoryUrls(IEnumerable pipelines) - { - return pipelines.Where(x => x.Repository?.Properties?.ContainsKey("manageUrl") == true) - .Select(x => x.Repository.Properties["manageUrl"]) - .Distinct() - .ToArray(); - } - } -} diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Azure.Sdk.Tools.PipelineOwnersExtractor.csproj b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Azure.Sdk.Tools.PipelineOwnersExtractor.csproj new file mode 100644 index 00000000000..c2fff291567 --- /dev/null +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Azure.Sdk.Tools.PipelineOwnersExtractor.csproj @@ -0,0 +1,25 @@ + + + + Exe + net6.0 + true + pipeline-owners-extractor + + + + + + + + + + + + + + Always + + + + diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/ISecretClientProvider.cs b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/ISecretClientProvider.cs new file mode 100644 index 00000000000..bf56d88701c --- /dev/null +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/ISecretClientProvider.cs @@ -0,0 +1,11 @@ +using System; + +using Azure.Security.KeyVault.Secrets; + +namespace Azure.Sdk.Tools.PipelineOwnersExtractor.Configuration +{ + public interface ISecretClientProvider + { + SecretClient GetSecretClient(Uri vaultUri); + } +} \ No newline at end of file diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/PipelineOwnerSettings.cs b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/PipelineOwnerSettings.cs new file mode 100644 index 00000000000..2cd0d05cd5a --- /dev/null +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/PipelineOwnerSettings.cs @@ -0,0 +1,19 @@ +namespace Azure.Sdk.Tools.PipelineOwnersExtractor.Configuration +{ + public class PipelineOwnerSettings + { + public string Account { get; set; } + + public string Projects { get; set; } + + public string OpenSourceAadAppId { get; set; } + + public string OpenSourceAadSecret { get; set; } + + public string OpenSourceAadTenantId { get; set; } + + public string AzureDevOpsPat { get; set; } + + public string Output { get; set; } + } +} diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/PostConfigureKeyVaultSettings.cs b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/PostConfigureKeyVaultSettings.cs new file mode 100644 index 00000000000..1cb78569f71 --- /dev/null +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/PostConfigureKeyVaultSettings.cs @@ -0,0 +1,60 @@ +using System; +using System.Linq; +using System.Text.RegularExpressions; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Azure.Sdk.Tools.PipelineOwnersExtractor.Configuration +{ + public class PostConfigureKeyVaultSettings : IPostConfigureOptions where T : class + { + private static readonly Regex secretRegex = new Regex(@"(?https://.*?\.vault\.azure\.net)/secrets/(?.*)", RegexOptions.Compiled, TimeSpan.FromSeconds(5)); + private readonly ILogger logger; + private readonly ISecretClientProvider secretClientProvider; + + public PostConfigureKeyVaultSettings(ILogger> logger, ISecretClientProvider secretClientProvider) + { + this.logger = logger; + this.secretClientProvider = secretClientProvider; + } + + public void PostConfigure(string name, T options) + { + var stringProperties = typeof(T) + .GetProperties() + .Where(x => x.PropertyType == typeof(string)); + + foreach (var property in stringProperties) + { + var value = (string)property.GetValue(options); + + if (value != null) + { + var match = secretRegex.Match(value); + + if (match.Success) + { + var vaultUrl = match.Groups["vault"].Value; + var secretName = match.Groups["secret"].Value; + + try + { + var secretClient = this.secretClientProvider.GetSecretClient(new Uri(vaultUrl)); + this.logger.LogInformation("Replacing setting property {PropertyName} with value from secret {SecretUrl}", property.Name, value); + + var response = secretClient.GetSecret(secretName); + var secret = response.Value; + + property.SetValue(options, secret.Value); + } + catch (Exception exception) + { + this.logger.LogError(exception, "Unable to read secret {SecretName} from vault {VaultUrl}", secretName, vaultUrl); + } + } + } + } + } + } +} diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/SecretClientProvider.cs b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/SecretClientProvider.cs new file mode 100644 index 00000000000..656c457a4f3 --- /dev/null +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/SecretClientProvider.cs @@ -0,0 +1,22 @@ +using System; + +using Azure.Core; +using Azure.Security.KeyVault.Secrets; + +namespace Azure.Sdk.Tools.PipelineOwnersExtractor.Configuration +{ + public class SecretClientProvider : ISecretClientProvider + { + private readonly TokenCredential tokenCredential; + + public SecretClientProvider(TokenCredential tokenCredential) + { + this.tokenCredential = tokenCredential; + } + + public SecretClient GetSecretClient(Uri vaultUri) + { + return new SecretClient(vaultUri, this.tokenCredential); + } + } +} \ No newline at end of file diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Extensions.cs b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Extensions.cs new file mode 100644 index 00000000000..67f2e56a5e2 --- /dev/null +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Extensions.cs @@ -0,0 +1,48 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.PipelineOwnersExtractor +{ + public static class Extensions + { + public static async Task LimitConcurrencyAsync(this IEnumerable> tasks, int concurrencyLimit = 1) + { + if (concurrencyLimit == int.MaxValue) + { + return await Task.WhenAll(tasks); + } + + var results = new List(); + + if (concurrencyLimit == 1) + { + foreach (var task in tasks) + { + results.Add(await task); + } + + return results.ToArray(); + } + + var pending = new List>(); + + foreach (var task in tasks) + { + pending.Add(task); + + if (pending.Count < concurrencyLimit) + { + continue; + } + + var completed = await Task.WhenAny(pending); + pending.Remove(completed); + results.Add(await completed); + } + + results.AddRange(await Task.WhenAll(pending)); + + return results.ToArray(); + } + } +} diff --git a/tools/pipeline-codeowners-extractor/GitHubService.cs b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/GitHubService.cs similarity index 71% rename from tools/pipeline-codeowners-extractor/GitHubService.cs rename to tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/GitHubService.cs index 866b215b011..c12ca0c75da 100644 --- a/tools/pipeline-codeowners-extractor/GitHubService.cs +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/GitHubService.cs @@ -1,23 +1,23 @@ -using Microsoft.Extensions.Logging; -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Threading.Tasks; using Azure.Sdk.Tools.CodeOwnersParser; +using Microsoft.Extensions.Logging; -namespace Azure.Sdk.Tools.PipelineCodeownerExtractor +namespace Azure.Sdk.Tools.PipelineOwnersExtractor { /// /// Interface for interacting with GitHub /// public class GitHubService { - private static HttpClient httpClient = new HttpClient(); - private static ConcurrentDictionary> codeownersFileCache = new ConcurrentDictionary>(); + private static readonly HttpClient httpClient = new HttpClient(); private readonly ILogger logger; + private readonly ConcurrentDictionary> codeOwnersFileCache; /// /// Creates a new GitHubService @@ -26,6 +26,7 @@ public class GitHubService public GitHubService(ILogger logger) { this.logger = logger; + this.codeOwnersFileCache = new ConcurrentDictionary>(); } /// @@ -33,16 +34,16 @@ public GitHubService(ILogger logger) /// /// GitHub repository URL /// Contents fo the located CODEOWNERS file - public async Task> GetCodeownersFile(Uri repoUrl) + public async Task> GetCodeOwnersFile(Uri repoUrl) { List result; - if (codeownersFileCache.TryGetValue(repoUrl.ToString(), out result)) + if (codeOwnersFileCache.TryGetValue(repoUrl.ToString(), out result)) { return result; } result = await GetCodeownersFileImpl(repoUrl); - codeownersFileCache.TryAdd(repoUrl.ToString(), result); + codeOwnersFileCache.TryAdd(repoUrl.ToString(), result); return result; } @@ -61,11 +62,11 @@ private async Task> GetCodeownersFileImpl(Uri repoUrl) var result = await httpClient.GetAsync(codeOwnersUrl); if (result.IsSuccessStatusCode) { - logger.LogInformation("Retrieved CODEOWNERS file URL = {0}", codeOwnersUrl); + this.logger.LogInformation("Retrieved CODEOWNERS file URL = {0}", codeOwnersUrl); return CodeOwnersFile.ParseContent(await result.Content.ReadAsStringAsync()); } - logger.LogWarning("Could not retrieve CODEOWNERS file URL = {0} ResponseCode = {1}", codeOwnersUrl, result.StatusCode); + this.logger.LogWarning("Could not retrieve CODEOWNERS file URL = {0} ResponseCode = {1}", codeOwnersUrl, result.StatusCode); return default; } diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Processor.cs b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Processor.cs new file mode 100644 index 00000000000..4a928e1b59f --- /dev/null +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Processor.cs @@ -0,0 +1,158 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +using Azure.Sdk.Tools.CodeOwnersParser; +using Azure.Sdk.Tools.NotificationConfiguration.Helpers; +using Azure.Sdk.Tools.NotificationConfiguration.Services; +using Azure.Sdk.Tools.PipelineOwnersExtractor.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.TeamFoundation.Build.WebApi; + +using Newtonsoft.Json; + +namespace Azure.Sdk.Tools.PipelineOwnersExtractor +{ + public class Processor + { + // Type 2 maps to a pipeline YAML file in the repository + private const int PipelineYamlProcessType = 2; + + private readonly ILogger logger; + private readonly GitHubToAADConverter githubToAadResolver; + private readonly AzureDevOpsService devOpsService; + private readonly GitHubService gitHubService; + private readonly PipelineOwnerSettings settings; + + public Processor( + ILogger logger, + GitHubService gitHubService, + GitHubToAADConverter githubToAadResolver, + AzureDevOpsService devOpsService, + IOptions options) + { + this.logger = logger; + this.gitHubService = gitHubService; + this.githubToAadResolver = githubToAadResolver; + this.devOpsService = devOpsService; + this.settings = options.Value; + } + + public async Task ExecuteAsync(CancellationToken stoppingToken = default) + { + var projects = this.settings.Projects.Split(','); + var pipelineResults = await Task.WhenAll(projects.Select(x => devOpsService.GetPipelinesAsync(x.Trim()))); + + // flatten arrays of pipelines by project into an array of pipelines + var pipelines = pipelineResults + .SelectMany(x => x) + .ToArray(); + + var repositoryUrls = GetDistinctRepositoryUrls(pipelines); + + var codeOwnerEntriesByRepository = await GetCodeOwnerEntriesAsync(repositoryUrls); + + var pipelineOwners = await AssociateOwnersToPipelinesAsync(pipelines, codeOwnerEntriesByRepository); + + var outputContent = pipelineOwners.ToDictionary(x => x.Pipeline.Id, x => x.Owners); + + await File.WriteAllTextAsync(this.settings.Output, JsonConvert.SerializeObject(outputContent, Formatting.Indented), stoppingToken); + } + + private async Task Owners)>> AssociateOwnersToPipelinesAsync( + IEnumerable pipelines, + Dictionary> codeOwnerEntriesByRepository) + { + var linkedGithubUsers = await githubToAadResolver.GetPeopleLinksAsync(); + + var microsoftAliasMap = linkedGithubUsers.ToDictionary(x => x.GitHub.Login, x => x.Aad.UserPrincipalName, StringComparer.OrdinalIgnoreCase); + + var microsoftPipelineOwners = new List<(BuildDefinition Pipeline, List Owners)>(); + + var unrecognizedGitHubAliases = new HashSet(); + + foreach (var pipeline in pipelines) + { + if (pipeline.Process.Type != PipelineYamlProcessType || !(pipeline.Process is YamlProcess process)) + { + logger.LogInformation("Skipping non-yaml pipeline '{Pipeline}'", pipeline.Name); + continue; + } + + if (pipeline.Repository.Type != "GitHub") + { + logger.LogInformation("Skipping pipeline '{Pipeline}' with {Type} repository", pipeline.Name, pipeline.Repository.Type); + continue; + } + + if (!codeOwnerEntriesByRepository.TryGetValue(SanitizeRepositoryUrl(pipeline.Repository.Url.AbsoluteUri), out var codeOwnerEntries)) + { + logger.LogInformation("Skipping pipeline '{Pipeline}' because its repo has no CODEOWNERS file", pipeline.Name); + continue; + } + + logger.LogInformation("Processing pipeline '{Pipeline}'", pipeline.Name); + + logger.LogInformation("Searching CODEOWNERS for patch matching {Path}", process.YamlFilename); + var codeOwnerEntry = CodeOwnersFile.FindOwnersForClosestMatch(codeOwnerEntries, process.YamlFilename); + codeOwnerEntry.FilterOutNonUserAliases(); + + logger.LogInformation("Matching Path = {Path}, Owner Count = {OwnerCount}", process.YamlFilename, codeOwnerEntry.Owners.Count); + + // Get set of team members in the CODEOWNERS file + var githubOwners = codeOwnerEntry.Owners.ToArray(); + + var microsoftOwners = new List(); + + foreach (var githubOwner in githubOwners) + { + if (microsoftAliasMap.TryGetValue(githubOwner, out var microsoftOwner)) + { + microsoftOwners.Add(microsoftOwner); + } + else + { + unrecognizedGitHubAliases.Add(githubOwner); + } + } + + microsoftPipelineOwners.Add((pipeline, microsoftOwners)); + } + + var mappedCount = microsoftPipelineOwners.SelectMany(x => x.Owners).Distinct().Count(); + logger.LogInformation("{Mapped} unique pipeline owner aliases mapped to Microsoft users. {Unmapped} could not be mapped.", mappedCount, unrecognizedGitHubAliases.Count); + + return microsoftPipelineOwners; + } + + private async Task>> GetCodeOwnerEntriesAsync(string[] repositoryUrls) + { + var tasks = repositoryUrls + .Select(SanitizeRepositoryUrl) + .Select(async url => ( + RepositoryUrl: url, + CodeOwners: await this.gitHubService.GetCodeOwnersFile(new Uri(url)) + )); + + var taskResults = await Task.WhenAll(tasks); + + return taskResults.Where(x => x.CodeOwners != null) + .ToDictionary(x => x.RepositoryUrl, x => x.CodeOwners, StringComparer.OrdinalIgnoreCase); + } + + private static string SanitizeRepositoryUrl(string url) => Regex.Replace(url, @"\.git$", string.Empty); + + private static string[] GetDistinctRepositoryUrls(IEnumerable pipelines) + { + return pipelines.Where(x => x.Repository?.Properties?.ContainsKey("manageUrl") == true) + .Select(x => x.Repository.Properties["manageUrl"]) + .Distinct() + .ToArray(); + } + } +} diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Program.cs b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Program.cs new file mode 100644 index 00000000000..07cc6c3f4de --- /dev/null +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Program.cs @@ -0,0 +1,68 @@ +using System; +using System.Threading.Tasks; +using Azure.Core; +using Azure.Identity; +using Azure.Sdk.Tools.NotificationConfiguration.Helpers; +using Azure.Sdk.Tools.NotificationConfiguration.Services; +using Azure.Sdk.Tools.PipelineOwnersExtractor.Configuration; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.VisualStudio.Services.Common; +using Microsoft.VisualStudio.Services.WebApi; + +namespace Azure.Sdk.Tools.PipelineOwnersExtractor +{ + public class Program + { + public static async Task Main(string[] args) + { + Console.WriteLine("Initializing PipelineOwnersExtractor"); + + using var host = Host.CreateDefaultBuilder(args) + .ConfigureServices((context, services) => + { + services.AddSingleton(); + services.AddSingleton(); + services.Configure(context.Configuration); + services.AddSingleton, PostConfigureKeyVaultSettings>(); + services.AddSingleton(); + services.AddSingleton(CreateGithubAadConverter); + services.AddSingleton(CreateAzureDevOpsService); + services.AddSingleton(); + }) + .Build(); + + var processor = host.Services.GetRequiredService(); + + await processor.ExecuteAsync(); + } + + private static AzureDevOpsService CreateAzureDevOpsService(IServiceProvider provider) + { + var logger = provider.GetRequiredService>(); + var settings = provider.GetRequiredService>().Value; + + var uri = new Uri($"https://dev.azure.com/{settings.Account}"); + var credentials = new VssBasicCredential("pat", settings.AzureDevOpsPat); + var connection = new VssConnection(uri, credentials); + + return new AzureDevOpsService(connection, logger); + } + + private static GitHubToAADConverter CreateGithubAadConverter(IServiceProvider provider) + { + var logger = provider.GetRequiredService>(); + var settings = provider.GetRequiredService>().Value; + + var credential = new ClientSecretCredential( + settings.OpenSourceAadTenantId, + settings.OpenSourceAadAppId, + settings.OpenSourceAadSecret); + + return new GitHubToAADConverter(credential, logger); + } + } +} diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/appsettings.json b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/appsettings.json new file mode 100644 index 00000000000..c5bd54b8af4 --- /dev/null +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/appsettings.json @@ -0,0 +1,17 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Warning", + "Microsoft.Hosting": "Information", + "Azure.Sdk.Tools": "Information", + "Azure.Core": "Error" + } + }, + "Account": "azure-sdk", + "Projects": "internal,public", + "OpenSourceAadAppId": "https://AzureSDKEngKeyVault.vault.azure.net/secrets/opensource-aad-app-id", + "OpenSourceAadSecret": "https://AzureSDKEngKeyVault.vault.azure.net/secrets/opensource-aad-secret", + "OpenSourceAadTenantId": "https://AzureSDKEngKeyVault.vault.azure.net/secrets/opensource-aad-tenant-id", + "AzureDevOpsPat": "https://AzureSDKEngKeyVault.vault.azure.net/secrets/azure-sdk-notification-tools-pat", + "Output": "pipelineOwners.json" +} diff --git a/tools/pipeline-codeowners-extractor/Azure.Sdk.Tools.PipelineCodeownerExtractor.sln b/tools/pipeline-owners-extractor/PipelineOwnersExtractor.sln similarity index 81% rename from tools/pipeline-codeowners-extractor/Azure.Sdk.Tools.PipelineCodeownerExtractor.sln rename to tools/pipeline-owners-extractor/PipelineOwnersExtractor.sln index 96bddd81631..cc454c2528c 100644 --- a/tools/pipeline-codeowners-extractor/Azure.Sdk.Tools.PipelineCodeownerExtractor.sln +++ b/tools/pipeline-owners-extractor/PipelineOwnersExtractor.sln @@ -3,22 +3,18 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.2.32602.215 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.PipelineCodeownerExtractor", "Azure.Sdk.Tools.PipelineCodeownerExtractor.csproj", "{863E8073-6BB4-4135-A30E-E3BE0141B3D7}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "identity-resolution", "..\identity-resolution\identity-resolution.csproj", "{54513498-7FA1-4525-915B-405746FBDDF5}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersParser", "..\code-owners-parser\CodeOwnersParser\Azure.Sdk.Tools.CodeOwnersParser.csproj", "{F0516B1D-037F-4096-9932-ED8F180EBC5D}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.PipelineOwnersExtractor", "Azure.Sdk.Tools.PipelineOwnersExtractor\Azure.Sdk.Tools.PipelineOwnersExtractor.csproj", "{4813A8E4-BC83-4F2D-8C87-F9AAAA11B207}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {863E8073-6BB4-4135-A30E-E3BE0141B3D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {863E8073-6BB4-4135-A30E-E3BE0141B3D7}.Debug|Any CPU.Build.0 = Debug|Any CPU - {863E8073-6BB4-4135-A30E-E3BE0141B3D7}.Release|Any CPU.ActiveCfg = Release|Any CPU - {863E8073-6BB4-4135-A30E-E3BE0141B3D7}.Release|Any CPU.Build.0 = Release|Any CPU {54513498-7FA1-4525-915B-405746FBDDF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {54513498-7FA1-4525-915B-405746FBDDF5}.Debug|Any CPU.Build.0 = Debug|Any CPU {54513498-7FA1-4525-915B-405746FBDDF5}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -27,6 +23,10 @@ Global {F0516B1D-037F-4096-9932-ED8F180EBC5D}.Debug|Any CPU.Build.0 = Debug|Any CPU {F0516B1D-037F-4096-9932-ED8F180EBC5D}.Release|Any CPU.ActiveCfg = Release|Any CPU {F0516B1D-037F-4096-9932-ED8F180EBC5D}.Release|Any CPU.Build.0 = Release|Any CPU + {4813A8E4-BC83-4F2D-8C87-F9AAAA11B207}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4813A8E4-BC83-4F2D-8C87-F9AAAA11B207}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4813A8E4-BC83-4F2D-8C87-F9AAAA11B207}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4813A8E4-BC83-4F2D-8C87-F9AAAA11B207}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/tools/pipeline-owners-extractor/ci.yml b/tools/pipeline-owners-extractor/ci.yml new file mode 100644 index 00000000000..dcef7d951b7 --- /dev/null +++ b/tools/pipeline-owners-extractor/ci.yml @@ -0,0 +1,27 @@ +# NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file. +trigger: + branches: + include: + - main + - feature/* + - release/* + - hotfix/* + paths: + include: + - tools/pipeline-owners-extractor + +pr: + branches: + include: + - main + - feature/* + - release/* + - hotfix/* + paths: + include: + - tools/pipeline-owners-extractor + +extends: + template: /eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml + parameters: + ToolDirectory: tools/pipeline-owners-extractor