diff --git a/eng/packages/General.props b/eng/packages/General.props index bd1948f3d26..e4a33591276 100644 --- a/eng/packages/General.props +++ b/eng/packages/General.props @@ -1,6 +1,7 @@ + diff --git a/eng/packages/TestOnly.props b/eng/packages/TestOnly.props index e9fa63dc4b6..8fce31e704e 100644 --- a/eng/packages/TestOnly.props +++ b/eng/packages/TestOnly.props @@ -3,7 +3,6 @@ - diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs index 2bea0ed0efd..08f035d55eb 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs @@ -1,10 +1,14 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.IO; using System.Threading; using System.Threading.Tasks; +using Azure.Identity; +using Azure.Storage.Files.DataLake; using Microsoft.Extensions.AI.Evaluation.Console.Utilities; +using Microsoft.Extensions.AI.Evaluation.Reporting; using Microsoft.Extensions.AI.Evaluation.Reporting.Storage; using Microsoft.Extensions.Logging; @@ -12,13 +16,29 @@ namespace Microsoft.Extensions.AI.Evaluation.Console.Commands; internal sealed class CleanCacheCommand(ILogger logger) { - internal async Task InvokeAsync(DirectoryInfo storageRootDir, CancellationToken cancellationToken = default) + internal async Task InvokeAsync(DirectoryInfo? storageRootDir, Uri? endpointUri, CancellationToken cancellationToken = default) { - string storageRootPath = storageRootDir.FullName; - logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath); - logger.LogInformation("Deleting expired cache entries..."); + IResponseCacheProvider cacheProvider; - var cacheProvider = new DiskBasedResponseCacheProvider(storageRootPath); + if (storageRootDir is not null) + { + string storageRootPath = storageRootDir.FullName; + logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath); + logger.LogInformation("Deleting expired cache entries..."); + + cacheProvider = new DiskBasedResponseCacheProvider(storageRootPath); + } + else if (endpointUri is not null) + { + logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri); + + var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential()); + cacheProvider = new AzureStorageResponseCacheProvider(fsClient); + } + else + { + throw new InvalidOperationException("Either --path or --endpoint must be specified"); + } await logger.ExecuteWithCatchAsync( () => cacheProvider.DeleteExpiredCacheEntriesAsync(cancellationToken)).ConfigureAwait(false); diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs index 9489e5b6e92..59635dc0530 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs @@ -1,11 +1,15 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; +using Azure.Identity; +using Azure.Storage.Files.DataLake; using Microsoft.Extensions.AI.Evaluation.Console.Utilities; +using Microsoft.Extensions.AI.Evaluation.Reporting; using Microsoft.Extensions.AI.Evaluation.Reporting.Storage; using Microsoft.Extensions.Logging; @@ -14,14 +18,31 @@ namespace Microsoft.Extensions.AI.Evaluation.Console.Commands; internal sealed class CleanResultsCommand(ILogger logger) { internal async Task InvokeAsync( - DirectoryInfo storageRootDir, + DirectoryInfo? storageRootDir, + Uri? endpointUri, int lastN, CancellationToken cancellationToken = default) { - string storageRootPath = storageRootDir.FullName; - logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath); + IResultStore resultStore; - var resultStore = new DiskBasedResultStore(storageRootPath); + if (storageRootDir is not null) + { + string storageRootPath = storageRootDir.FullName; + logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath); + + resultStore = new DiskBasedResultStore(storageRootPath); + } + else if (endpointUri is not null) + { + logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri); + + var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential()); + resultStore = new AzureStorageResultStore(fsClient); + } + else + { + throw new InvalidOperationException("Either --path or --endpoint must be specified"); + } await logger.ExecuteWithCatchAsync( async ValueTask () => diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs index ec7f659edd0..7d21dd11a87 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs @@ -3,9 +3,12 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Tasks; +using Azure.Identity; +using Azure.Storage.Files.DataLake; using Microsoft.Extensions.AI.Evaluation.Reporting; using Microsoft.Extensions.AI.Evaluation.Reporting.Formats.Html; using Microsoft.Extensions.AI.Evaluation.Reporting.Formats.Json; @@ -17,17 +20,36 @@ namespace Microsoft.Extensions.AI.Evaluation.Console.Commands; internal sealed partial class ReportCommand(ILogger logger) { internal async Task InvokeAsync( - DirectoryInfo storageRootDir, + DirectoryInfo? storageRootDir, + Uri? endpointUri, FileInfo outputFile, + bool openReport, int lastN, Format format, CancellationToken cancellationToken = default) { - string storageRootPath = storageRootDir.FullName; - logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath); + IResultStore resultStore; - var results = new List(); - var resultStore = new DiskBasedResultStore(storageRootPath); + if (storageRootDir is not null) + { + string storageRootPath = storageRootDir.FullName; + logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath); + + resultStore = new DiskBasedResultStore(storageRootPath); + } + else if (endpointUri is not null) + { + logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri); + + var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential()); + resultStore = new AzureStorageResultStore(fsClient); + } + else + { + throw new InvalidOperationException("Either --path or --endpoint must be specified"); + } + + List results = []; await foreach (string executionName in resultStore.GetLatestExecutionNamesAsync(lastN, cancellationToken).ConfigureAwait(false)) @@ -38,6 +60,8 @@ internal async Task InvokeAsync( cancellationToken: cancellationToken).ConfigureAwait(false)) { results.Add(result); + + logger.LogInformation("Execution: {executionName} Scenario: {scenarioName} Iteration: {iterationName}", result.ExecutionName, result.ScenarioName, result.IterationName); } } @@ -58,6 +82,24 @@ internal async Task InvokeAsync( await reportWriter.WriteReportAsync(results, cancellationToken).ConfigureAwait(false); logger.LogInformation("Report: {outputFilePath} [{format}]", outputFilePath, format); + // See the following issues for reasoning behind this check. We want to avoid opening the report + // if this process is running as a service or in a CI pipeline. + // https://github.com/dotnet/runtime/issues/770#issuecomment-564700467 + // https://github.com/dotnet/runtime/issues/66530#issuecomment-1065854289 + bool isRedirected = System.Console.IsInputRedirected && System.Console.IsOutputRedirected && System.Console.IsErrorRedirected; + bool isInteractive = Environment.UserInteractive && (OperatingSystem.IsWindows() || !(isRedirected)); + + if (openReport && isInteractive) + { + // Open the generated report in the default browser. + _ = Process.Start( + new ProcessStartInfo + { + FileName = outputFilePath, + UseShellExecute = true + }); + } + return 0; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj index 9198f92d801..624f1984425 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj @@ -24,6 +24,8 @@ + + @@ -31,6 +33,7 @@ + diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Program.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Program.cs index 5158dbe4262..056c71e7b80 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Program.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Program.cs @@ -2,10 +2,11 @@ // The .NET Foundation licenses this file to you under the MIT license. #if DEBUG -using System.CommandLine.Parsing; using System.Diagnostics; #endif +using System; using System.CommandLine; +using System.CommandLine.Parsing; using System.IO; using System.Threading.Tasks; using Microsoft.Extensions.AI.Evaluation.Console.Commands; @@ -15,6 +16,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Console; internal sealed class Program { + private const string ShortName = "aieval"; private const string Name = "Microsoft.Extensions.AI.Evaluation.Console"; private const string Banner = $"{Name} [{Constants.Version}]"; @@ -23,7 +25,7 @@ private static async Task Main(string[] args) #pragma warning restore EA0014 { using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddConsole()); - ILogger logger = factory.CreateLogger(Name); + ILogger logger = factory.CreateLogger(ShortName); logger.LogInformation("{banner}", Banner); var rootCmd = new RootCommand(Banner); @@ -33,19 +35,54 @@ private static async Task Main(string[] args) rootCmd.AddGlobalOption(debugOpt); #endif - var reportCmd = new Command("report", "Generate a report "); + var reportCmd = new Command("report", "Generate a report from a result store"); var pathOpt = new Option( ["-p", "--path"], "Root path under which the cache and results are stored") { - IsRequired = true + IsRequired = false }; + var endpointOpt = + new Option( + ["--endpoint"], + "Endpoint URL under which the cache and results are stored for Azure Data Lake Gen2 storage") + { + IsRequired = false + }; + + var openReportOpt = + new Option( + ["--open"], + getDefaultValue: () => false, + "Open the report in the default browser") + { + IsRequired = false, + }; + + ValidateSymbolResult requiresPathOrEndpoint = (CommandResult cmd) => + { + bool hasPath = cmd.GetValueForOption(pathOpt) is not null; + bool hasEndpoint = cmd.GetValueForOption(endpointOpt) is not null; + if (!(hasPath ^ hasEndpoint)) + { + cmd.ErrorMessage = $"Either '{pathOpt.Name}' or '{endpointOpt.Name}' must be specified."; + } + }; + reportCmd.AddOption(pathOpt); + reportCmd.AddOption(endpointOpt); + reportCmd.AddOption(openReportOpt); + reportCmd.AddValidator(requiresPathOrEndpoint); - var outputOpt = new Option(["-o", "--output"], "Output filename/path") { IsRequired = true }; + var outputOpt = new Option( + ["-o", "--output"], + "Output filename/path") + { + IsRequired = true, + }; reportCmd.AddOption(outputOpt); var lastNOpt = new Option(["-n"], () => 1, "Number of most recent executions to include in the report."); @@ -60,9 +97,11 @@ private static async Task Main(string[] args) reportCmd.AddOption(formatOpt); reportCmd.SetHandler( - (path, output, lastN, format) => new ReportCommand(logger).InvokeAsync(path, output, lastN, format), + (path, endpoint, output, openReport, lastN, format) => new ReportCommand(logger).InvokeAsync(path, endpoint, output, openReport, lastN, format), pathOpt, + endpointOpt, outputOpt, + openReportOpt, lastNOpt, formatOpt); @@ -70,27 +109,32 @@ private static async Task Main(string[] args) // TASK: Support more granular filters such as the specific scenario / iteration / execution whose results must // be cleaned up. - var cleanResults = new Command("cleanResults", "Delete results"); - cleanResults.AddOption(pathOpt); + var cleanResultsCmd = new Command("cleanResults", "Delete results"); + cleanResultsCmd.AddOption(pathOpt); + cleanResultsCmd.AddOption(endpointOpt); + cleanResultsCmd.AddValidator(requiresPathOrEndpoint); var lastNOpt2 = new Option(["-n"], () => 0, "Number of most recent executions to preserve."); - cleanResults.AddOption(lastNOpt2); + cleanResultsCmd.AddOption(lastNOpt2); - cleanResults.SetHandler( - (path, lastN) => new CleanResultsCommand(logger).InvokeAsync(path, lastN), + cleanResultsCmd.SetHandler( + (path, endpoint, lastN) => new CleanResultsCommand(logger).InvokeAsync(path, endpoint, lastN), pathOpt, + endpointOpt, lastNOpt2); - rootCmd.Add(cleanResults); + rootCmd.Add(cleanResultsCmd); - var cleanCache = new Command("cleanCache", "Delete expired cache entries"); - cleanCache.AddOption(pathOpt); + var cleanCacheCmd = new Command("cleanCache", "Delete expired cache entries"); + cleanCacheCmd.AddOption(pathOpt); + cleanCacheCmd.AddOption(endpointOpt); + cleanCacheCmd.AddValidator(requiresPathOrEndpoint); - cleanCache.SetHandler( - path => new CleanCacheCommand(logger).InvokeAsync(path), - pathOpt); + cleanCacheCmd.SetHandler( + (path, endpoint) => new CleanCacheCommand(logger).InvokeAsync(path, endpoint), + pathOpt, endpointOpt); - rootCmd.Add(cleanCache); + rootCmd.Add(cleanCacheCmd); // TASK: Support some mechanism to fail a build (i.e. return a failure exit code) based on one or more user // specified criteria (e.g., if x% of metrics were deemed 'poor'). Ideally this mechanism would be flexible / @@ -106,4 +150,5 @@ private static async Task Main(string[] args) return await rootCmd.InvokeAsync(args).ConfigureAwait(false); } + } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/Directory.Build.targets b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/Directory.Build.targets index 87375c68d6e..1aa66cd8fcb 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/Directory.Build.targets +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/Directory.Build.targets @@ -67,7 +67,7 @@ internal static class Constants - + <_VSIXPackageVersionFile>$(MSBuildThisFileDirectory)\TypeScript\azure-devops-report\VSIXPackageVersion.json