From 723daf990783f6a15f2e7490ef1effa4c0fe2633 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Mon, 5 Dec 2022 15:01:27 -0800 Subject: [PATCH 01/16] Ensure `push` of an unrestored `assets.json` is a recognizable error (#4868) * and set environment exit code to -1 --- .../Azure.Sdk.Tools.TestProxy/Store/GitStore.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs index df96bdc5469..94a540815f4 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs @@ -85,6 +85,18 @@ public async Task GetPath(string pathToAssetsJson) /// public async Task Push(string pathToAssetsJson) { var config = await ParseConfigurationFile(pathToAssetsJson); + + var initialized = IsAssetsRepoInitialized(config); + + if (!initialized) + { + _consoleWrapper.WriteLine($"The targeted assets.json \"{config.AssetsJsonRelativeLocation}\" has not been restored prior to attempting push. " + + $"Are you certain you're pushing the correct assets.json? Please invoke \'test-proxy restore \"{config.AssetsJsonRelativeLocation}\"\' prior to invoking a push operation."); + + Environment.ExitCode = -1; + return; + } + var pendingChanges = DetectPendingChanges(config); var generatedTagName = config.TagPrefix; From 2356b06ca32da4eebb0429c868a95666170731e4 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Mon, 5 Dec 2022 15:47:44 -0800 Subject: [PATCH 02/16] [Perf] Add profiling to perf automation (#4878) - Rebased against main - Taking in favor of #4034 --- eng/common/pipelines/templates/jobs/perf.yml | 15 +++++++++ .../Azure.Sdk.Tools.PerfAutomation/Cpp.cs | 2 +- .../ILanguage.cs | 2 +- .../Azure.Sdk.Tools.PerfAutomation/Java.cs | 20 ++++++++++-- .../JavaScript.cs | 2 +- .../LanguageBase.cs | 5 ++- .../Azure.Sdk.Tools.PerfAutomation/Net.cs | 2 +- .../Azure.Sdk.Tools.PerfAutomation/Program.cs | 32 +++++++++++++++++-- .../Azure.Sdk.Tools.PerfAutomation/Python.cs | 2 +- tools/perf-automation/tests.yml | 8 +++++ 10 files changed, 80 insertions(+), 10 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/perf.yml b/eng/common/pipelines/templates/jobs/perf.yml index ecd794a1f61..b19bd5c88bd 100644 --- a/eng/common/pipelines/templates/jobs/perf.yml +++ b/eng/common/pipelines/templates/jobs/perf.yml @@ -53,6 +53,9 @@ parameters: - name: Iterations type: number default: '5' +- name: Profile + type: boolean + default: false - name: AdditionalArguments type: string default: '' @@ -77,6 +80,11 @@ jobs: MatrixName: 'Windows' variables: - ${{ parameters.Variables }} + - name: Profile + ${{ if parameters.Profile }}: + value: '--profile' + ${{ else }}: + value: '' pool: name: $(Pool) vmImage: $(OSVmImage) @@ -125,6 +133,7 @@ jobs: --tests "${{ parameters.Tests }}" --arguments "${{ parameters.Arguments }}" --iterations ${{ parameters.Iterations }} + $(Profile) ${{ parameters.AdditionalArguments }} workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation env: @@ -162,6 +171,12 @@ jobs: artifactName: results-${{ parameters.Language }}-$(MatrixName) condition: always() + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(System.DefaultWorkingDirectory)/${{ parameters.Language }}-profile.zip + artifactName: ${{ parameters.Language }}-profile.zip + condition: ${{ parameters.Profile }} + - template: /eng/common/TestResources/remove-test-resources.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Cpp.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Cpp.cs index 2e6f7e0f40c..76d0a4a92ed 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Cpp.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Cpp.cs @@ -58,7 +58,7 @@ await Util.RunAsync( } public override async Task RunAsync(string project, string languageVersion, - string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context) + string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context, bool profile) { var perfExe = (string)context; diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/ILanguage.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/ILanguage.cs index 692f1243c70..944e087b667 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/ILanguage.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/ILanguage.cs @@ -7,7 +7,7 @@ namespace Azure.Sdk.Tools.PerfAutomation public interface ILanguage { Task<(string output, string error, object context)> SetupAsync(string project, string languageVersion, string primaryPackage, IDictionary packageVersions); - Task RunAsync(string project, string languageVersion, string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context); + Task RunAsync(string project, string languageVersion, string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context, bool profile); Task CleanupAsync(string project); IDictionary FilterRuntimePackageVersions(IDictionary runtimePackageVersions); } diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Java.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Java.cs index 57ba20eb6e6..47739fc96aa 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Java.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Java.cs @@ -19,6 +19,8 @@ public class Java : LanguageBase private string PerfCoreProjectFile => Path.Combine(WorkingDirectory, "common", "perf-test-core", "pom.xml"); private string VersionFile => Path.Combine(WorkingDirectory, "eng", "versioning", "version_client.txt"); + private static int profileCount = 0; + private static readonly Dictionary _buildEnvironment = new Dictionary() { // Prevents error "InvocationTargetException: Java heap space" in azure-storage-file-datalake when compiling azure-storage-perf @@ -98,7 +100,7 @@ private static void UpdatePackageVersions(string projectFile, IDictionary RunAsync(string project, string languageVersion, - string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context) + string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context, bool profile) { var jarFile = (string)context; @@ -109,7 +111,21 @@ public override async Task RunAsync(string project, string lang outputBuilder: outputBuilder, errorBuilder: errorBuilder); var runtimePackageVersions = GetRuntimePackageVersions(dependencyListResult.StandardOutput); - var processArguments = $"-XX:+CrashOnOutOfMemoryError -jar {jarFile} -- {testName} {arguments}"; + // '-XX:+UnlockCommercialFeatures' isn't required and fails when used in a version higher than Java 8, need to inspect the Java version to determine if it should be added. + var profilingConfig = ""; + if (profile) + { + var profileOutputPath = Path.GetFullPath(Path.Combine(ProfileDirectory, $"{testName}_{profileCount++}.jfr")); + profilingConfig = $"-XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=filename={profileOutputPath},maxsize=1gb"; + + var jfrConfigurationFile = Path.Combine(WorkingDirectory, "eng", "PerfAutomation.jfc"); + if (File.Exists(jfrConfigurationFile)) + { + profilingConfig += $",settings={jfrConfigurationFile}"; + } + } + + var processArguments = $"-XX:+CrashOnOutOfMemoryError {profilingConfig} -jar {jarFile} -- {testName} {arguments}"; var result = await Util.RunAsync("java", processArguments, WorkingDirectory, throwOnError: false, outputBuilder: outputBuilder, errorBuilder: errorBuilder); diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/JavaScript.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/JavaScript.cs index 40eb26b6832..345e3e3ff35 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/JavaScript.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/JavaScript.cs @@ -121,7 +121,7 @@ await Util.RunAsync("node", $"{_rush} deploy -p {projectName}", } public override async Task RunAsync(string project, string languageVersion, - string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context) + string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context, bool profile) { var runtimePackageVersions = (Dictionary) context; diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/LanguageBase.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/LanguageBase.cs index 4bae3709aa1..dcf8fc4b1de 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/LanguageBase.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/LanguageBase.cs @@ -1,5 +1,6 @@ using Azure.Sdk.Tools.PerfAutomation.Models; using System.Collections.Generic; +using System.IO; using System.Threading.Tasks; namespace Azure.Sdk.Tools.PerfAutomation @@ -8,6 +9,7 @@ public abstract class LanguageBase : ILanguage { protected abstract Language Language { get; } protected string WorkingDirectory => Program.Config.WorkingDirectories[Language]; + protected string ProfileDirectory => Path.GetFullPath(Path.Combine(WorkingDirectory, Language + "-profile")); public abstract Task CleanupAsync(string project); @@ -18,7 +20,8 @@ public abstract Task RunAsync( IDictionary packageVersions, string testName, string arguments, - object context); + object context, + bool profile); public abstract Task<(string output, string error, object context)> SetupAsync( string project, diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs index f2ff8946169..2d966a6805a 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs @@ -84,7 +84,7 @@ public class Net : LanguageBase } public override async Task RunAsync(string project, string languageVersion, - string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context) + string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context, bool profile) { var dllName = Path.GetFileNameWithoutExtension(project) + ".dll"; var dllPath = Path.Combine(PublishDirectory, dllName); diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Program.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Program.cs index 520610d122c..29fc6d01745 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Program.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Program.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Text.Encodings.Web; using System.Text.Json; @@ -85,6 +86,9 @@ public class RunOptions [Option('p', "package-versions", HelpText = "Regex of package versions to run")] public string PackageVersions { get; set; } + [Option("profile", HelpText = "Enables capture of profiling data")] + public bool Profile { get; set; } + [Option('s', "services", HelpText = "Regex of services to run")] public string Services { get; set; } @@ -217,6 +221,7 @@ private static async Task Run(RunOptions options) var outputMd = outputFiles[3]; var results = new List(); + var profileDirectories = new List(); foreach (var service in selectedServices) { @@ -225,6 +230,18 @@ private static async Task Run(RunOptions options) var language = l.Key; var serviceLanugageInfo = l.Value; + if (options.Profile) + { + // For each language create a directory name "{language name}-profile" that will be used to contain + // all profiling data for a performance run by that language. + // Later this directory will be zipped to create ZIP file that can be retained with the name "{language name}-profile.zip". + string profileDirectory = Path.Combine(Program.Config.WorkingDirectories[language], language + "-profile"); + if (!Directory.Exists(profileDirectory)) + { + profileDirectories.Add(Directory.CreateDirectory(profileDirectory)); + } + } + var languageInfo = selectedlanguages[language]; foreach (var languageVersion in languageInfo.DefaultVersions.Concat(languageInfo.OptionalVersions)) @@ -238,6 +255,16 @@ await RunPackageVersion(options, outputJson, outputCsv, outputTxt, outputMd, res } } } + + if (options.Profile) + { + // For each language that ran create a ZIP file containing all profiling data collected. + // This can be retained for in-depth performance analysis. + foreach (var profileDirectory in profileDirectories) + { + ZipFile.CreateFromDirectory(profileDirectory.FullName, Path.Combine(profileDirectory.Parent.FullName, profileDirectory.Name + ".zip")); + } + } } private static async Task RunPackageVersion(RunOptions options, string outputJson, string outputCsv, string outputTxt, @@ -348,7 +375,7 @@ private static async Task RunPackageVersion(RunOptions options, string outputJso try { Console.WriteLine($"RunAsync({serviceLanguageInfo.Project}, {languageVersion}, " + - $"{test.TestNames[language]}, {allArguments}, {context})"); + $"{test.TestNames[language]}, {allArguments}, {context}, {options.Profile})"); Console.WriteLine(); iterationResult = await _languages[language].RunAsync( @@ -358,7 +385,8 @@ private static async Task RunPackageVersion(RunOptions options, string outputJso packageVersions, test.TestNames[language], allArguments, - context + context, + options.Profile ); } catch (Exception e) diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Python.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Python.cs index b26c132c4d9..3d46853b267 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Python.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Python.cs @@ -66,7 +66,7 @@ public class Python : LanguageBase } public override async Task RunAsync(string project, string languageVersion, - string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context) + string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context, bool profile) { var projectDirectory = Path.Combine(WorkingDirectory, project); diff --git a/tools/perf-automation/tests.yml b/tools/perf-automation/tests.yml index 7e377a97540..4f7135c71c3 100644 --- a/tools/perf-automation/tests.yml +++ b/tools/perf-automation/tests.yml @@ -64,6 +64,9 @@ parameters: - name: Iterations type: number default: 1 +- name: Profile + type: boolean + default: false - name: OperatingSystems type: string default: 'Linux' @@ -137,6 +140,7 @@ stages: Tests: ${{ parameters.Tests }} Arguments: ${{ parameters.Arguments }} Iterations: ${{ parameters.Iterations }} + Profile: ${{ parameters.Profile }} - ${{ if parameters.IncludeJava }}: - template: /eng/common/pipelines/templates/jobs/perf.yml @@ -170,6 +174,7 @@ stages: Tests: ${{ parameters.Tests }} Arguments: ${{ parameters.Arguments }} Iterations: ${{ parameters.Iterations }} + Profile: ${{ parameters.Profile }} - ${{ if parameters.IncludeJS }}: - template: /eng/common/pipelines/templates/jobs/perf.yml @@ -196,6 +201,7 @@ stages: Tests: ${{ parameters.Tests }} Arguments: ${{ parameters.Arguments }} Iterations: ${{ parameters.Iterations }} + Profile: ${{ parameters.Profile }} - ${{ if parameters.IncludePython }}: - template: /eng/common/pipelines/templates/jobs/perf.yml @@ -225,6 +231,7 @@ stages: Tests: ${{ parameters.Tests }} Arguments: ${{ parameters.Arguments }} Iterations: ${{ parameters.Iterations }} + Profile: ${{ parameters.Profile }} - ${{ if parameters.IncludeCpp }}: - template: /eng/common/pipelines/templates/jobs/perf.yml @@ -251,6 +258,7 @@ stages: Tests: ${{ parameters.Tests }} Arguments: ${{ parameters.Arguments }} Iterations: ${{ parameters.Iterations }} + Profile: ${{ parameters.Profile }} - stage: Print_Results displayName: Print Results From ec375264dd89dd4f35315667ee4c8036ff858901 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Mon, 5 Dec 2022 17:15:22 -0800 Subject: [PATCH 03/16] [Perf] Move "profile" parameter before "context" (#4884) - Split long argument lists onto multiple lines - Alphabetize members in ILanguage and LanguageBase --- .../Azure.Sdk.Tools.PerfAutomation/Cpp.cs | 16 +++++++++++--- .../ILanguage.cs | 19 +++++++++++++++-- .../Azure.Sdk.Tools.PerfAutomation/Java.cs | 21 +++++++++++++++---- .../JavaScript.cs | 16 +++++++++++--- .../LanguageBase.cs | 14 +++++++------ .../Azure.Sdk.Tools.PerfAutomation/Net.cs | 16 +++++++++++--- .../Azure.Sdk.Tools.PerfAutomation/Program.cs | 4 ++-- .../Azure.Sdk.Tools.PerfAutomation/Python.cs | 16 +++++++++++--- 8 files changed, 96 insertions(+), 26 deletions(-) diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Cpp.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Cpp.cs index 76d0a4a92ed..df8db37c5e8 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Cpp.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Cpp.cs @@ -26,7 +26,10 @@ public class Cpp : LanguageBase protected override Language Language => Language.Cpp; public override async Task<(string output, string error, object context)> SetupAsync( - string project, string languageVersion, string primaryPackage, IDictionary packageVersions) + string project, + string languageVersion, + string primaryPackage, + IDictionary packageVersions) { var buildDirectory = Path.Combine(WorkingDirectory, _buildDirectory); @@ -57,8 +60,15 @@ await Util.RunAsync( return (result.StandardOutput, result.StandardError, exe); } - public override async Task RunAsync(string project, string languageVersion, - string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context, bool profile) + public override async Task RunAsync( + string project, + string languageVersion, + string primaryPackage, + IDictionary packageVersions, + string testName, + string arguments, + bool profile, + object context) { var perfExe = (string)context; diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/ILanguage.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/ILanguage.cs index 944e087b667..ba3d10b39bb 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/ILanguage.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/ILanguage.cs @@ -6,9 +6,24 @@ namespace Azure.Sdk.Tools.PerfAutomation { public interface ILanguage { - Task<(string output, string error, object context)> SetupAsync(string project, string languageVersion, string primaryPackage, IDictionary packageVersions); - Task RunAsync(string project, string languageVersion, string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context, bool profile); Task CleanupAsync(string project); + IDictionary FilterRuntimePackageVersions(IDictionary runtimePackageVersions); + + Task<(string output, string error, object context)> SetupAsync( + string project, + string languageVersion, + string primaryPackage, + IDictionary packageVersions); + + Task RunAsync( + string project, + string languageVersion, + string primaryPackage, + IDictionary packageVersions, + string testName, + string arguments, + bool profile, + object context); } } diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Java.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Java.cs index 47739fc96aa..cbfeb24254c 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Java.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Java.cs @@ -28,7 +28,10 @@ public class Java : LanguageBase }; public override async Task<(string output, string error, object context)> SetupAsync( - string project, string languageVersion, string primaryPackage, IDictionary packageVersions) + string project, + string languageVersion, + string primaryPackage, + IDictionary packageVersions) { var projectFile = Path.Combine(WorkingDirectory, project, "pom.xml"); @@ -58,7 +61,10 @@ public class Java : LanguageBase return (result.StandardOutput, result.StandardError, jar); } - private static void UpdatePackageVersions(string projectFile, IDictionary packageVersions, IDictionary sourceVersions) + private static void UpdatePackageVersions( + string projectFile, + IDictionary packageVersions, + IDictionary sourceVersions) { // Create backup. Throw if exists, since this shouldn't happen File.Copy(projectFile, projectFile + ".bak", overwrite: false); @@ -99,8 +105,15 @@ private static void UpdatePackageVersions(string projectFile, IDictionary RunAsync(string project, string languageVersion, - string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context, bool profile) + public override async Task RunAsync( + string project, + string languageVersion, + string primaryPackage, + IDictionary packageVersions, + string testName, + string arguments, + bool profile, + object context) { var jarFile = (string)context; diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/JavaScript.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/JavaScript.cs index 345e3e3ff35..1c7d24a98c4 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/JavaScript.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/JavaScript.cs @@ -17,7 +17,10 @@ public class JavaScript : LanguageBase protected override Language Language => Language.JS; public override async Task<(string output, string error, object context)> SetupAsync( - string project, string languageVersion, string primaryPackage, IDictionary packageVersions) + string project, + string languageVersion, + string primaryPackage, + IDictionary packageVersions) { var outputBuilder = new StringBuilder(); var errorBuilder = new StringBuilder(); @@ -120,8 +123,15 @@ await Util.RunAsync("node", $"{_rush} deploy -p {projectName}", return (outputBuilder.ToString(), errorBuilder.ToString(), runtimePackageVersions); } - public override async Task RunAsync(string project, string languageVersion, - string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context, bool profile) + public override async Task RunAsync( + string project, + string languageVersion, + string primaryPackage, + IDictionary packageVersions, + string testName, + string arguments, + bool profile, + object context) { var runtimePackageVersions = (Dictionary) context; diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/LanguageBase.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/LanguageBase.cs index dcf8fc4b1de..e976bf68505 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/LanguageBase.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/LanguageBase.cs @@ -8,11 +8,16 @@ namespace Azure.Sdk.Tools.PerfAutomation public abstract class LanguageBase : ILanguage { protected abstract Language Language { get; } - protected string WorkingDirectory => Program.Config.WorkingDirectories[Language]; + protected string ProfileDirectory => Path.GetFullPath(Path.Combine(WorkingDirectory, Language + "-profile")); + protected string WorkingDirectory => Program.Config.WorkingDirectories[Language]; + public abstract Task CleanupAsync(string project); + public virtual IDictionary FilterRuntimePackageVersions(IDictionary runtimePackageVersions) + => runtimePackageVersions; + public abstract Task RunAsync( string project, string languageVersion, @@ -20,16 +25,13 @@ public abstract Task RunAsync( IDictionary packageVersions, string testName, string arguments, - object context, - bool profile); + bool profile, + object context); public abstract Task<(string output, string error, object context)> SetupAsync( string project, string languageVersion, string primaryPackage, IDictionary packageVersions); - - public virtual IDictionary FilterRuntimePackageVersions(IDictionary runtimePackageVersions) - => runtimePackageVersions; } } diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs index 2d966a6805a..e378a6a11d2 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs @@ -16,7 +16,10 @@ public class Net : LanguageBase private string PublishDirectory => Path.Join(WorkingDirectory, "artifacts", "perf"); public override async Task<(string output, string error, object context)> SetupAsync( - string project, string languageVersion, string primaryPackage, IDictionary packageVersions) + string project, + string languageVersion, + string primaryPackage, + IDictionary packageVersions) { var projectFile = Path.Combine(WorkingDirectory, project); @@ -83,8 +86,15 @@ public class Net : LanguageBase return (result.StandardOutput, result.StandardError, null); } - public override async Task RunAsync(string project, string languageVersion, - string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context, bool profile) + public override async Task RunAsync( + string project, + string languageVersion, + string primaryPackage, + IDictionary packageVersions, + string testName, + string arguments, + bool profile, + object context) { var dllName = Path.GetFileNameWithoutExtension(project) + ".dll"; var dllPath = Path.Combine(PublishDirectory, dllName); diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Program.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Program.cs index 29fc6d01745..0f1f5d021f3 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Program.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Program.cs @@ -385,8 +385,8 @@ private static async Task RunPackageVersion(RunOptions options, string outputJso packageVersions, test.TestNames[language], allArguments, - context, - options.Profile + options.Profile, + context ); } catch (Exception e) diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Python.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Python.cs index 3d46853b267..373eeaac330 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Python.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Python.cs @@ -17,7 +17,10 @@ public class Python : LanguageBase protected override Language Language => Language.Python; public override async Task<(string output, string error, object context)> SetupAsync( - string project, string languageVersion, string primaryPackage, IDictionary packageVersions) + string project, + string languageVersion, + string primaryPackage, + IDictionary packageVersions) { var projectDirectory = Path.Combine(WorkingDirectory, project); var env = Path.Combine(projectDirectory, _env); @@ -65,8 +68,15 @@ public class Python : LanguageBase return (outputBuilder.ToString(), errorBuilder.ToString(), null); } - public override async Task RunAsync(string project, string languageVersion, - string primaryPackage, IDictionary packageVersions, string testName, string arguments, object context, bool profile) + public override async Task RunAsync( + string project, + string languageVersion, + string primaryPackage, + IDictionary packageVersions, + string testName, + string arguments, + bool profile, + object context) { var projectDirectory = Path.Combine(WorkingDirectory, project); From 23703d23e922257be2723625c8af8b16dc66a116 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Mon, 5 Dec 2022 19:02:56 -0800 Subject: [PATCH 04/16] we renamed the tool from azure.sdk.tools.testproxy to test-proxy (#4869) --- eng/pipelines/templates/steps/test-proxy-local-tool.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/test-proxy-local-tool.yml b/eng/pipelines/templates/steps/test-proxy-local-tool.yml index c340f15cf0f..77199c54678 100644 --- a/eng/pipelines/templates/steps/test-proxy-local-tool.yml +++ b/eng/pipelines/templates/steps/test-proxy-local-tool.yml @@ -9,7 +9,7 @@ steps: dotnet tool install --tool-path $(Build.BinariesDirectory)/test-proxy ` --prerelease ` --add-source $(Build.ArtifactStagingDirectory) ` - azure.sdk.tools.testproxy + test-proxy displayName: "Install test-proxy from local file" workingDirectory: $(Build.SourcesDirectory)/tools/test-proxy From 85818590096a71e2ae72aefda7053aaaaa981df1 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 6 Dec 2022 11:01:48 -0800 Subject: [PATCH 05/16] reverse check for common order (#4889) --- tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs index 94a540815f4..c2b6deb87cc 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs @@ -634,7 +634,7 @@ public DirectoryEvaluation EvaluateDirectory(string directoryPath) return new DirectoryEvaluation() { AssetsJsonPresent = File.Exists(assetsJsonLocation), - IsGitRoot = File.Exists(gitLocation) || Directory.Exists(gitLocation), + IsGitRoot = Directory.Exists(gitLocation) || File.Exists(gitLocation), IsRoot = new DirectoryInfo(directoryPath).Parent == null }; } From 189316ae60ecbaf88f079f520bf32036b03c361a Mon Sep 17 00:00:00 2001 From: Konrad Jamrozik Date: Tue, 6 Dec 2022 14:53:48 -0800 Subject: [PATCH 06/16] Assorted set of minor refactorings for logic related to CODEOWNERS processing (#4885) See https://github.com/Azure/azure-sdk-tools/pull/4885#issue-1478101966 --- .../ConsoleOutput.cs | 8 +++---- .../MainTests.cs | 17 +++++++------- .../Program.cs | 6 ++--- tools/code-owners-parser/CodeOwnersParser.sln | 8 ++++++- .../CodeOwnersParser.sln.DotSettings | 4 ++++ .../CodeOwnersParser/CodeOwnerEntry.cs | 3 +-- .../CodeOwnersParser/CodeOwnersFile.cs | 23 +++++-------------- .../CodeOwnersParser/FileHelpers.cs | 2 +- .../Services/GitHubService.cs | 8 ++++--- ...notification-configuration.sln.DotSettings | 3 +++ 10 files changed, 42 insertions(+), 40 deletions(-) create mode 100644 tools/code-owners-parser/CodeOwnersParser.sln.DotSettings create mode 100644 tools/notification-configuration/notification-configuration.sln.DotSettings diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/ConsoleOutput.cs b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/ConsoleOutput.cs index 28d68a082c0..a4de98d7690 100644 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/ConsoleOutput.cs +++ b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/ConsoleOutput.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; namespace Azure.Sdk.Tools.RetrieveCodeOwners.Tests @@ -8,8 +8,8 @@ namespace Azure.Sdk.Tools.RetrieveCodeOwners.Tests /// public class ConsoleOutput : IDisposable { - private StringWriter stringWriter; - private TextWriter originalOutput; + private readonly StringWriter stringWriter; + private readonly TextWriter originalOutput; /// /// The constructor is where we take in the console output and output to string writer. @@ -25,7 +25,7 @@ public ConsoleOutput() /// Writes the text representation of a string builder to the string. /// /// The string from console output. - public string GetOuput() + public string GetOutput() { return this.stringWriter.ToString(); } diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/MainTests.cs b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/MainTests.cs index 10b4d8231d3..6242a046ec2 100644 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/MainTests.cs +++ b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/MainTests.cs @@ -11,9 +11,9 @@ namespace Azure.Sdk.Tools.RetrieveCodeOwners.Tests [TestFixture] public class MainTests { - private const string codeOwnerFilePath = "CODEOWNERS"; + private const string CodeOwnersFilePath = "CODEOWNERS"; - private static readonly object[] _sourceLists = + private static readonly object[] sourceLists = { new object[] {"sdk", false, new List { "person1", "person2" } }, new object[] { "/sdk", false, new List { "person1", "person2" } }, @@ -24,14 +24,13 @@ public class MainTests new object[] { "/sd", true, new List() } }; - [TestCaseSource("_sourceLists")] - public void TestOnNormalOuput(string targetDirectory, bool includeUserAliasesOnly, List expectedReturn) + [TestCaseSource(nameof(sourceLists))] + public void TestOnNormalOutput(string targetDirectory, bool includeUserAliasesOnly, List expectedReturn) { - using (var consoleOutput = new ConsoleOutput()) { - Program.Main(codeOwnerFilePath, targetDirectory, includeUserAliasesOnly); - var output = consoleOutput.GetOuput(); + Program.Main(CodeOwnersFilePath, targetDirectory, includeUserAliasesOnly); + var output = consoleOutput.GetOutput(); TestExpectResult(expectedReturn, output); consoleOutput.Dispose(); } @@ -45,10 +44,10 @@ public void TestOnError(string codeOwnerPath) Assert.AreEqual(1, Program.Main(codeOwnerPath, "sdk")); } - private void TestExpectResult(List expectReturn, string output) + private static void TestExpectResult(List expectReturn, string output) { CodeOwnerEntry codeOwnerEntry = JsonSerializer.Deserialize(output); - List actualReturn = codeOwnerEntry.Owners; + List actualReturn = codeOwnerEntry!.Owners; Assert.AreEqual(expectReturn.Count, actualReturn.Count); for (int i = 0; i < actualReturn.Count; i++) { diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs index 6c94f8c678f..cb16e53bd3b 100644 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs +++ b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Text.Json; using Azure.Sdk.Tools.CodeOwnersParser; @@ -7,10 +7,10 @@ namespace Azure.Sdk.Tools.RetrieveCodeOwners /// /// The tool command to retrieve code owners. /// - public class Program + public static class Program { /// - /// Retrieves codeowners information for specific section of the repo + /// Retrieves CODEOWNERS information for specific section of the repo /// /// The path of CODEOWNERS file in repo /// The directory whose information is to be retrieved diff --git a/tools/code-owners-parser/CodeOwnersParser.sln b/tools/code-owners-parser/CodeOwnersParser.sln index 54d20fc7b39..260c393ac31 100644 --- a/tools/code-owners-parser/CodeOwnersParser.sln +++ b/tools/code-owners-parser/CodeOwnersParser.sln @@ -7,7 +7,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersP EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.RetrieveCodeOwners", "Azure.Sdk.Tools.RetrieveCodeOwners\Azure.Sdk.Tools.RetrieveCodeOwners.csproj", "{FE65F92D-C71B-4E38-A4B2-3089EA7C5FEC}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.Sdk.Tools.RetrieveCodeOwners.Tests", "Azure.Sdk.Tools.RetrieveCodeOwners.Tests\Azure.Sdk.Tools.RetrieveCodeOwners.Tests.csproj", "{798B8CAC-68FC-49FD-A0F6-51C0DC4A4D1D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.RetrieveCodeOwners.Tests", "Azure.Sdk.Tools.RetrieveCodeOwners.Tests\Azure.Sdk.Tools.RetrieveCodeOwners.Tests.csproj", "{798B8CAC-68FC-49FD-A0F6-51C0DC4A4D1D}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{EF585CCA-55F8-44EB-921A-30996CBAFC49}" + ProjectSection(SolutionItems) = preProject + ci.yml = ci.yml + ..\..\eng\common\scripts\get-codeowners.ps1 = ..\..\eng\common\scripts\get-codeowners.ps1 + EndProjectSection EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/tools/code-owners-parser/CodeOwnersParser.sln.DotSettings b/tools/code-owners-parser/CodeOwnersParser.sln.DotSettings new file mode 100644 index 00000000000..1414c94c6aa --- /dev/null +++ b/tools/code-owners-parser/CodeOwnersParser.sln.DotSettings @@ -0,0 +1,4 @@ + + PR + True + True \ No newline at end of file diff --git a/tools/code-owners-parser/CodeOwnersParser/CodeOwnerEntry.cs b/tools/code-owners-parser/CodeOwnersParser/CodeOwnerEntry.cs index a81e07c7865..c2fa426734a 100644 --- a/tools/code-owners-parser/CodeOwnersParser/CodeOwnerEntry.cs +++ b/tools/code-owners-parser/CodeOwnersParser/CodeOwnerEntry.cs @@ -1,7 +1,6 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; -using System.Net.Http; namespace Azure.Sdk.Tools.CodeOwnersParser { diff --git a/tools/code-owners-parser/CodeOwnersParser/CodeOwnersFile.cs b/tools/code-owners-parser/CodeOwnersParser/CodeOwnersFile.cs index ce66a7ee108..578cd531e80 100644 --- a/tools/code-owners-parser/CodeOwnersParser/CodeOwnersFile.cs +++ b/tools/code-owners-parser/CodeOwnersParser/CodeOwnersFile.cs @@ -1,4 +1,3 @@ -using OutputColorizer; using System; using System.Collections.Generic; using System.IO; @@ -9,20 +8,15 @@ public static class CodeOwnersFile { public static List ParseFile(string filePathOrUrl) { - string content; - content = FileHelpers.GetFileContents(filePathOrUrl); - + string content = FileHelpers.GetFileContents(filePathOrUrl); return ParseContent(content); } public static List ParseContent(string fileContent) { List entries = new List(); - string line; - // An entry ends when we get to a path (a real path or a commented dummy path) - using (StringReader sr = new StringReader(fileContent)) { CodeOwnerEntry entry = new CodeOwnerEntry(); @@ -30,7 +24,7 @@ public static List ParseContent(string fileContent) // we are going to read line by line until we find a line that is not a comment OR that is using the placeholder entry inside the comment. // while we are trying to find the folder entry, we parse all comment lines to extract the labels from it. // when we find the path or placeholder, we add the completed entry and create a new one. - while ((line = sr.ReadLine()) != null) + while (sr.ReadLine() is { } line) { line = NormalizeLine(line); @@ -93,14 +87,9 @@ public static CodeOwnerEntry FindOwnersForClosestMatch(List code } private static string NormalizeLine(string line) - { - if (string.IsNullOrEmpty(line)) - { - return line; - } - - // Remove tabs and trim extra whitespace - return line.Replace('\t', ' ').Trim(); - } + => !string.IsNullOrEmpty(line) + // Remove tabs and trim extra whitespace + ? line.Replace('\t', ' ').Trim() + : line; } } diff --git a/tools/code-owners-parser/CodeOwnersParser/FileHelpers.cs b/tools/code-owners-parser/CodeOwnersParser/FileHelpers.cs index 97a078e2946..2c9b0ff58e2 100644 --- a/tools/code-owners-parser/CodeOwnersParser/FileHelpers.cs +++ b/tools/code-owners-parser/CodeOwnersParser/FileHelpers.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using System.Net.Http; diff --git a/tools/identity-resolution/Services/GitHubService.cs b/tools/identity-resolution/Services/GitHubService.cs index 29b48c4b588..bbd40d23e3f 100644 --- a/tools/identity-resolution/Services/GitHubService.cs +++ b/tools/identity-resolution/Services/GitHubService.cs @@ -1,4 +1,4 @@ -using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -14,8 +14,10 @@ namespace Azure.Sdk.Tools.NotificationConfiguration /// public class GitHubService { - private static HttpClient httpClient = new HttpClient(); - private static ConcurrentDictionary> codeownersFileCache = new ConcurrentDictionary>(); + private static readonly HttpClient httpClient = new HttpClient(); + + private static readonly ConcurrentDictionary> + codeownersFileCache = new ConcurrentDictionary>(); private readonly ILogger logger; diff --git a/tools/notification-configuration/notification-configuration.sln.DotSettings b/tools/notification-configuration/notification-configuration.sln.DotSettings new file mode 100644 index 00000000000..1e20132c27c --- /dev/null +++ b/tools/notification-configuration/notification-configuration.sln.DotSettings @@ -0,0 +1,3 @@ + + PR + True \ No newline at end of file From a0765e0f5f9fc1b4a6fe2663079e2c4c353843be Mon Sep 17 00:00:00 2001 From: Praven Kuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Wed, 7 Dec 2022 20:13:55 -0500 Subject: [PATCH 07/16] CADL manual API review (#4832) * Add support for manual CADL review gen --- eng/pipelines/apiview-review-gen-cadl.yml | 75 +++++ eng/scripts/Create-Apiview-Token-Cadl.ps1 | 98 +++++++ .../ReviewManagerTests.cs | 4 +- .../APIView/APIViewWeb/Client/src/reviews.ts | 276 ++++++++++-------- .../Languages/CadlLanguageService.cs | 67 +++++ .../APIViewWeb/Languages/LanguageService.cs | 10 + .../Languages/PythonLanguageService.cs | 2 - .../APIViewWeb/Managers/IReviewManager.cs | 4 +- .../APIViewWeb/Managers/ReviewManager.cs | 58 ++-- .../Models/ReviewGenPipelineParamModel.cs | 6 +- .../APIView/APIViewWeb/Models/UploadModel.cs | 12 +- .../APIViewWeb/Pages/Assemblies/Index.cshtml | 68 +++-- .../Pages/Assemblies/Index.cshtml.cs | 11 +- src/dotnet/APIView/APIViewWeb/Startup.cs | 1 + 14 files changed, 513 insertions(+), 179 deletions(-) create mode 100644 eng/pipelines/apiview-review-gen-cadl.yml create mode 100644 eng/scripts/Create-Apiview-Token-Cadl.ps1 create mode 100644 src/dotnet/APIView/APIViewWeb/Languages/CadlLanguageService.cs diff --git a/eng/pipelines/apiview-review-gen-cadl.yml b/eng/pipelines/apiview-review-gen-cadl.yml new file mode 100644 index 00000000000..f712b018c6c --- /dev/null +++ b/eng/pipelines/apiview-review-gen-cadl.yml @@ -0,0 +1,75 @@ +pr: none + +trigger: none + +parameters: + - name: Reviews + type: string + default: '[{"ReviewID":"","RevisionID":"","SourceRepoName":"","FileName":"","SourceBranchName":""}]' + - name: APIViewURL + type: string + default: 'https://apiview.dev' + +pool: + name: azsdk-pool-mms-ubuntu-2004-general + vmImage: MMSUbuntu20.04 + +variables: + NodeVersion: '16.x' + +jobs: +- job: CreateSwaggerReviewCodeFile + displayName: 'Create Swagger API review token file' + + variables: + - template: /eng/pipelines/templates/variables/globals.yml + + steps: + - task: NodeTool@0 + inputs: + versionSpec: $(NodeVersion) + displayName: "Use Node $(NodeVersion)" + + - pwsh: | + $reviews = "${{ parameters.Reviews }}" + Write-Host "Reviews: $($reviews)" + echo "##vso[task.setvariable variable=Reviews]$reviews" + displayName: "Setup Reviews Variable" + condition: eq(variables['Reviews'], '') + + - pwsh: | + $url = "${{parameters.APIViewURL}}" + echo "##vso[task.setvariable variable=APIViewURL]$url" + displayName: "Setup APIViewURL Variable" + condition: eq(variables['APIViewURL'], '') + + - pwsh: | + npm install -g cadl + npm install -g @cadl-lang/compiler + npm install -g @azure-tools/cadl-apiview + displayName: "Install npm packages" + + - task: Powershell@2 + displayName: 'Generate APIView Token files' + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/Create-Apiview-Token-Cadl.ps1 + arguments: > + -Reviews "$(Reviews)" + -OutputDir "$(Build.ArtifactStagingDirectory)" + -WorkingDir "$(Pipeline.Workspace)" + + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: 'apiview' + + - task: Powershell@2 + displayName: 'Send Request to APIView to Update Token files' + condition: succeededOrFailed() + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/Apiview-Update-Generated-Review.ps1 + arguments: > + -BuildId $(Build.BuildId) + -ApiviewUpdateUrl "$(APIViewURL)/review/UpdateApiReview" diff --git a/eng/scripts/Create-Apiview-Token-Cadl.ps1 b/eng/scripts/Create-Apiview-Token-Cadl.ps1 new file mode 100644 index 00000000000..c7fdc0a54f1 --- /dev/null +++ b/eng/scripts/Create-Apiview-Token-Cadl.ps1 @@ -0,0 +1,98 @@ +[CmdletBinding()] +param ( + [ValidateNotNullOrEmpty()] + [string] $Reviews, + [ValidateNotNullOrEmpty()] + [string] $WorkingDir, + [ValidateNotNullOrEmpty()] + [string] $OutputDir +) + +Set-StrictMode -Version 3 +$sparseCheckoutFile = ".git/info/sparse-checkout" + + +function Sparse-Checkout($branchName, $packagePath) +{ + Remove-Item $sparseCheckoutFile -Force + git sparse-checkout init --cone + git sparse-checkout set $packagePath + git checkout $BranchName +} + +function Generate-Apiview-File($packagePath) +{ + Write-Host "Generating API review token file from path '$($packagePath)'" + Push-Location $packagePath + try + { + npm install + cadl compile . --emit=@azure-tools/cadl-apiview + } + finally + { + Pop-Location + } +} + +function Stage-Apiview-File($packagePath, $reviewId, $revisionId) +{ + $tokenFilePath = Join-Path $packagePath "cadl-output" + $stagingReviewPath = Join-Path $OutputDir $reviewId + $stagingPath = Join-Path $stagingReviewPath $revisionId + Write-Host "Copying APIView file from '$($tokenFilePath)' to '$($stagingPath)'" + New-Item $stagingPath -ItemType Directory -Force + Copy-Item -Destination $stagingPath -Path "$tokenFilePath/*" +} + + +Write-Host "Review Details Json: $($Reviews)" +$revs = ConvertFrom-Json $Reviews +if ($revs) +{ + $prevRepo = "" + foreach ($r in $revs) + { + $reviewId = $r.ReviewID + $revisionId = $r.RevisionID + $packagePath = $r.FileName + $GitRepoName = $r.SourceRepoName + $branchName = $r.SourceBranchName + + Write-Host "Generating API review for Review ID: '$($reviewId), Revision ID: '$($revisionId)" + Write-Host "URL to Repo: '$($GitRepoName), Branch name: '$($branchName), Package Path: '$($packagePath)" + + $repoDirectory = Split-Path $GitRepoName -leaf + if (Test-Path $repoDirectory) + { + Write-Host "Destination path '$($repoDirectory)' already exists in working directory and is not an empty directory." + exit 1 + } + # initialize git clone if current review is generated from different repo than previous one + if ($GitRepoName -ne $prevRepo) + { + git clone --no-checkout --filter=tree:0 "https://github.com/$GitRepoName" + if ($LASTEXITCODE) { exit $LASTEXITCODE } + $prevRepo = $GitRepoName + } + + $repoDirectory = Split-Path $GitRepoName -leaf + Push-Location $repoDirectory + try + { + Write-Host "GitHub Repo Name: '$($repoDirectory)" + # Sparse checkout package root path + Sparse-Checkout -branchName $branchName -packagePath $packagePath + # Generate API code file + Generate-Apiview-File -packagePath $packagePath + #Copy generated code file to stagin location + Stage-Apiview-File -packagePath $packagePath -reviewId $reviewId -revisionId $revisionId + } + finally + { + Pop-Location + } + } + + Write-Host "Generated and copied Api review token file to output directory" +} \ No newline at end of file diff --git a/src/dotnet/APIView/APIViewIntegrationTests/ReviewManagerTests.cs b/src/dotnet/APIView/APIViewIntegrationTests/ReviewManagerTests.cs index 8172f120337..32bf5be634e 100644 --- a/src/dotnet/APIView/APIViewIntegrationTests/ReviewManagerTests.cs +++ b/src/dotnet/APIView/APIViewIntegrationTests/ReviewManagerTests.cs @@ -51,7 +51,7 @@ public async Task AddRevisionAsync_Computes_Headings_Of_Sections_With_Diff_A() { var reviewManager = testsBaseFixture.ReviewManager; var user = testsBaseFixture.User; - var review = await testsBaseFixture.ReviewManager.CreateReviewAsync(user, fileNameA, "Revision1", fileStreamA, false, true); + var review = await testsBaseFixture.ReviewManager.CreateReviewAsync(user, fileNameA, "Revision1", fileStreamA, false, "Swagger", true); await reviewManager.AddRevisionAsync(user, review.ReviewId, fileNameB, "Revision2", fileStreamB, true); review = await reviewManager.GetReviewAsync(user, review.ReviewId); var headingWithDiffInSections = review.Revisions[0].HeadingsOfSectionsWithDiff[review.Revisions[1].RevisionId]; @@ -64,7 +64,7 @@ public async Task AddRevisionAsync_Computes_Headings_Of_Sections_With_Diff_B() { var reviewManager = testsBaseFixture.ReviewManager; var user = testsBaseFixture.User; - var review = await reviewManager.CreateReviewAsync(user, fileNameC, "Azure.Analytics.Purview.Account", fileStreamC, false, true); + var review = await reviewManager.CreateReviewAsync(user, fileNameC, "Azure.Analytics.Purview.Account", fileStreamC, false, "Swagger", true); await reviewManager.AddRevisionAsync(user, review.ReviewId, fileNameD, "Azure.Analytics.Purview.Account", fileStreamD, true); review = await reviewManager.GetReviewAsync(user, review.ReviewId); var headingWithDiffInSections = review.Revisions[0].HeadingsOfSectionsWithDiff[review.Revisions[1].RevisionId]; diff --git a/src/dotnet/APIView/APIViewWeb/Client/src/reviews.ts b/src/dotnet/APIView/APIViewWeb/Client/src/reviews.ts index 6104dad3868..4e911b1a16d 100644 --- a/src/dotnet/APIView/APIViewWeb/Client/src/reviews.ts +++ b/src/dotnet/APIView/APIViewWeb/Client/src/reviews.ts @@ -1,129 +1,147 @@ -$(() => { - const defaultPageSize = 50; - const reviewsFilterPartial = $( '#reviews-filter-partial' ); - const languageFilter = $( '#language-filter-bootstraps-select' ); - const stateFilter = $( '#state-filter-bootstraps-select' ); - const statusFilter = $( '#status-filter-bootstraps-select' ); - const typeFilter = $( '#type-filter-bootstraps-select' ); - const searchBox = $( '#reviews-table-search-box' ); - const searchButton = $( '#reviews-search-button' ); - const resetButton = $( '#reset-filter-button' ); - - // Import underscorejs - var _ = require('underscore'); - - // Enable tooltip - ($('[data-toggle="tooltip"]')).tooltip(); - - // Computes the uri string using the values of search, pagination and various filters - // Invokes partial page update to list of reviews using ajax - // Updates the uri displayed on the client - function updateListedReviews({ pageNo = 1, pageSize = defaultPageSize } = {}) - { - var uri = '?handler=reviewspartial'; - var searchQuery = searchBox.val() as string; - - if (searchQuery != null && searchQuery.trim() != '') - { - var searchTerms = searchQuery.trim().split(/\s+/); - searchTerms.forEach(function(value, index){ - uri = uri + '&search=' + encodeURIComponent(value); - }); - } - - languageFilter.children(":selected").each(function() { - uri = uri + '&languages=' + encodeURIComponent(`${$(this).val()}`); - }); - - stateFilter.children(":selected").each(function() { - uri = uri + '&state=' + encodeURIComponent(`${$(this).val()}`); - }); - - statusFilter.children(":selected").each(function() { - uri = uri + '&status=' + encodeURIComponent(`${$(this).val()}`); - }); - - typeFilter.children(":selected").each(function() { - uri = uri + '&type=' + encodeURIComponent(`${$(this).val()}`); - }); - - uri = uri + '&pageNo=' + encodeURIComponent(pageNo); - uri = uri + '&pageSize=' + encodeURIComponent(pageSize); - uri = encodeURI(uri); - - $.ajax({ - url: uri - }).done(function(partialViewResult) { - reviewsFilterPartial.html(partialViewResult); - history.pushState({}, '', uri.replace('handler=reviewspartial&', '')); - addPaginationEventHandlers(); // This ensures that the event handlers are re-added after ajax refresh - }); - } - - // Add custom behaviour and event to pagination buttons - function addPaginationEventHandlers() - { - $( '.page-link' ).each(function() { - $(this).on('click', function(event){ - event.preventDefault(); - var linkParts = $(this).prop('href').split('/'); - var pageNo = linkParts[linkParts.length - 1]; - if (pageNo !== null && pageNo !== undefined) - { - updateListedReviews({ pageNo: pageNo }); - } - }); - }); - } - - // Fetches data for populating dropdown options - function updateFilterDropDown(filter, query) - { - var uri = `?handler=reviews${query}`; - var urlParams = new URLSearchParams(location.search); - if (urlParams.has(query)) - { - urlParams.getAll(query).forEach(function(value, index) { - uri = uri + `&selected${query}=` + encodeURIComponent(value); - }); - } - $.ajax({ - url: uri - }).done(function(partialViewResult) { - filter.html(partialViewResult); - (filter).selectpicker('refresh'); - }); - } - - // Fetch content of dropdown on page load - $(document).ready(function() { - updateFilterDropDown(languageFilter, "languages"); // Pulls languages data from DB - addPaginationEventHandlers(); - }); - - // Update list of reviews when any dropdown is changed - [languageFilter, stateFilter, statusFilter, typeFilter].forEach(function(value, index) { - value.on('hidden.bs.select', function() { - updateListedReviews(); - }); - }); - - // Update list of reviews based on search input - searchBox.on('input', _.debounce(function(e) { - updateListedReviews(); - }, 600)); - - searchButton.on('click', function() { - updateListedReviews(); - }); - - // Reset list of reviews as well as filters - resetButton.on('click', function(e) { - (languageFilter).selectpicker('deselectAll'); - (stateFilter).selectpicker('deselectAll').selectpicker('val', 'Open'); - (statusFilter).selectpicker('deselectAll'); - (typeFilter).selectpicker('deselectAll'); - searchBox.val(''); - updateListedReviews(); - }); -}); +$(() => { + const defaultPageSize = 50; + const reviewsFilterPartial = $( '#reviews-filter-partial' ); + const languageFilter = $( '#language-filter-bootstraps-select' ); + const stateFilter = $( '#state-filter-bootstraps-select' ); + const statusFilter = $( '#status-filter-bootstraps-select' ); + const typeFilter = $( '#type-filter-bootstraps-select' ); + const searchBox = $( '#reviews-table-search-box' ); + const searchButton = $( '#reviews-search-button' ); + const resetButton = $('#reset-filter-button'); + const languageSelect = $('#review-language-select'); + + // Import underscorejs + var _ = require('underscore'); + + // Enable tooltip + ($('[data-toggle="tooltip"]')).tooltip(); + + // Computes the uri string using the values of search, pagination and various filters + // Invokes partial page update to list of reviews using ajax + // Updates the uri displayed on the client + function updateListedReviews({ pageNo = 1, pageSize = defaultPageSize } = {}) + { + var uri = '?handler=reviewspartial'; + var searchQuery = searchBox.val() as string; + + if (searchQuery != null && searchQuery.trim() != '') + { + var searchTerms = searchQuery.trim().split(/\s+/); + searchTerms.forEach(function(value, index){ + uri = uri + '&search=' + encodeURIComponent(value); + }); + } + + languageFilter.children(":selected").each(function() { + uri = uri + '&languages=' + encodeURIComponent(`${$(this).val()}`); + }); + + stateFilter.children(":selected").each(function() { + uri = uri + '&state=' + encodeURIComponent(`${$(this).val()}`); + }); + + statusFilter.children(":selected").each(function() { + uri = uri + '&status=' + encodeURIComponent(`${$(this).val()}`); + }); + + typeFilter.children(":selected").each(function() { + uri = uri + '&type=' + encodeURIComponent(`${$(this).val()}`); + }); + + uri = uri + '&pageNo=' + encodeURIComponent(pageNo); + uri = uri + '&pageSize=' + encodeURIComponent(pageSize); + uri = encodeURI(uri); + + $.ajax({ + url: uri + }).done(function(partialViewResult) { + reviewsFilterPartial.html(partialViewResult); + history.pushState({}, '', uri.replace('handler=reviewspartial&', '')); + addPaginationEventHandlers(); // This ensures that the event handlers are re-added after ajax refresh + }); + } + + // Add custom behaviour and event to pagination buttons + function addPaginationEventHandlers() + { + $( '.page-link' ).each(function() { + $(this).on('click', function(event){ + event.preventDefault(); + var linkParts = $(this).prop('href').split('/'); + var pageNo = linkParts[linkParts.length - 1]; + if (pageNo !== null && pageNo !== undefined) + { + updateListedReviews({ pageNo: pageNo }); + } + }); + }); + } + + // Fetches data for populating dropdown options + function updateFilterDropDown(filter, query) + { + var uri = `?handler=reviews${query}`; + var urlParams = new URLSearchParams(location.search); + if (urlParams.has(query)) + { + urlParams.getAll(query).forEach(function(value, index) { + uri = uri + `&selected${query}=` + encodeURIComponent(value); + }); + } + $.ajax({ + url: uri + }).done(function(partialViewResult) { + filter.html(partialViewResult); + (filter).selectpicker('refresh'); + }); + } + + // Fetch content of dropdown on page load + $(document).ready(function() { + updateFilterDropDown(languageFilter, "languages"); // Pulls languages data from DB + addPaginationEventHandlers(); + }); + + // Update list of reviews when any dropdown is changed + [languageFilter, stateFilter, statusFilter, typeFilter].forEach(function(value, index) { + value.on('hidden.bs.select', function() { + updateListedReviews(); + }); + }); + + // Update list of reviews based on search input + searchBox.on('input', _.debounce(function(e) { + updateListedReviews(); + }, 600)); + + searchButton.on('click', function() { + updateListedReviews(); + }); + + // Reset list of reviews as well as filters + resetButton.on('click', function(e) { + (languageFilter).selectpicker('deselectAll'); + (stateFilter).selectpicker('deselectAll').selectpicker('val', 'Open'); + (statusFilter).selectpicker('deselectAll'); + (typeFilter).selectpicker('deselectAll'); + searchBox.val(''); + updateListedReviews(); + }); + + var prevLanguageValue = languageSelect.val(); + languageSelect.on('change', function (e) { + var val = $(this).val(); + if (val == "C++" || val == "C#") { + val = val.replace("C++", "Cpp").replace("C#", "Csharp"); + } + var helpName = "#help-" + val; + $(helpName).click(); + if (val == 'Cadl' || prevLanguageValue == 'Cadl') { + const fileSelectors = $(".package-selector"); + for (var i = 0; i < fileSelectors.length; i++) { + $(fileSelectors[i]).toggleClass("hidden-row"); + } + } + prevLanguageValue = val; + }); +}); diff --git a/src/dotnet/APIView/APIViewWeb/Languages/CadlLanguageService.cs b/src/dotnet/APIView/APIViewWeb/Languages/CadlLanguageService.cs new file mode 100644 index 00000000000..78405c5ac46 --- /dev/null +++ b/src/dotnet/APIView/APIViewWeb/Languages/CadlLanguageService.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using ApiView; +using APIViewWeb.Models; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json; + +namespace APIViewWeb +{ + public class CadlLanguageService : LanguageProcessor + { + public override string Name { get; } = "Cadl"; + + public override string Extension { get; } = ".cadl"; + + public override string VersionString { get; } = "0"; + + public override string ProcessName => throw new NotImplementedException(); + + private string _cadlSpecificPathPrefix; + + public CadlLanguageService(IConfiguration configuration) + { + IsReviewGenByPipeline = true; + _cadlSpecificPathPrefix = configuration["CADL-Specification-path-prefix"] ?? "/specification"; + } + public override async Task GetCodeFileAsync(string originalName, Stream stream, bool runAnalysis) + { + return await CodeFile.DeserializeAsync(stream, true); + } + + public override string GetProcessorArguments(string originalName, string tempDirectory, string jsonPath) + { + throw new NotImplementedException(); + } + public override bool CanUpdate(string versionString) + { + return false; + } + + public override bool GeneratePipelineRunParams(ReviewGenPipelineParamModel param) + { + var filePath = param.FileName.ToLower(); + // Verify cadl source file path is a GitHub URL to cadl package root + if (filePath == null || !filePath.StartsWith("https://github.com/")) + return false; + + if (!filePath.Contains("/tree/") || !filePath.Contains(_cadlSpecificPathPrefix)) + return false; + + filePath = filePath.Replace("https://github.com/", ""); + var sourceUrlparts = filePath.Split("/tree/", 2); + param.SourceRepoName = sourceUrlparts[0]; + sourceUrlparts = sourceUrlparts[1].Split(_cadlSpecificPathPrefix, 2); + param.SourceBranchName = sourceUrlparts[0]; + param.FileName = $"{_cadlSpecificPathPrefix}{sourceUrlparts[1]}"; + _telemetryClient.TrackTrace($"Pipeline parameters to run CADL API rview gen pipeline: '{JsonConvert.SerializeObject(param)}'"); + + return true; + } + } +} diff --git a/src/dotnet/APIView/APIViewWeb/Languages/LanguageService.cs b/src/dotnet/APIView/APIViewWeb/Languages/LanguageService.cs index 7fb53cc2a01..8a634825910 100644 --- a/src/dotnet/APIView/APIViewWeb/Languages/LanguageService.cs +++ b/src/dotnet/APIView/APIViewWeb/Languages/LanguageService.cs @@ -4,6 +4,9 @@ using System.Threading.Tasks; using ApiView; using APIView; +using APIViewWeb.Models; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.ApplicationInsights; namespace APIViewWeb { @@ -25,5 +28,12 @@ public abstract class LanguageService Tokens = new CodeFileToken[] {new CodeFileToken("", CodeFileTokenKind.Newline), ReviewNotReadyCodeFile, new CodeFileToken("", CodeFileTokenKind.Newline) }, Navigation = new NavigationItem[] { new NavigationItem() { Text = fileName } } }; + + public static string[] SupportedLanguages = new string[] { "C", "C++", "C#", "Cadl", "Go", "Java", "JavaScript", "Json", "Kotlin", "Python", "Swagger", "Swift", "Xml" }; + + public virtual bool GeneratePipelineRunParams(ReviewGenPipelineParamModel param) => true; + + + public static TelemetryClient _telemetryClient = new(TelemetryConfiguration.CreateDefault()); } } diff --git a/src/dotnet/APIView/APIViewWeb/Languages/PythonLanguageService.cs b/src/dotnet/APIView/APIViewWeb/Languages/PythonLanguageService.cs index 4a3d004ddcb..f5857224c04 100644 --- a/src/dotnet/APIView/APIViewWeb/Languages/PythonLanguageService.cs +++ b/src/dotnet/APIView/APIViewWeb/Languages/PythonLanguageService.cs @@ -22,8 +22,6 @@ public class PythonLanguageService : LanguageProcessor private readonly string _pythonExecutablePath; public override string ProcessName => _pythonExecutablePath; - static TelemetryClient _telemetryClient = new TelemetryClient(TelemetryConfiguration.CreateDefault()); - public PythonLanguageService(IConfiguration configuration) { _pythonExecutablePath = configuration["PYTHONEXECUTABLEPATH"] ?? "python"; diff --git a/src/dotnet/APIView/APIViewWeb/Managers/IReviewManager.cs b/src/dotnet/APIView/APIViewWeb/Managers/IReviewManager.cs index f3eecb7286e..546dd41fc02 100644 --- a/src/dotnet/APIView/APIViewWeb/Managers/IReviewManager.cs +++ b/src/dotnet/APIView/APIViewWeb/Managers/IReviewManager.cs @@ -11,7 +11,7 @@ namespace APIViewWeb.Managers { public interface IReviewManager { - public Task CreateReviewAsync(ClaimsPrincipal user, string originalName, string label, Stream fileStream, bool runAnalysis, bool awaitComputeDiff = false); + public Task CreateReviewAsync(ClaimsPrincipal user, string originalName, string label, Stream fileStream, bool runAnalysis, string langauge, bool awaitComputeDiff = false); public Task> GetReviewsAsync(bool closed, string language, string packageName = null, ReviewType filterType = ReviewType.Manual); public Task> GetReviewsAsync(string ServiceName, string PackageName, IEnumerable filterTypes); public Task> GetReviewPropertiesAsync(string propertyName); @@ -21,7 +21,7 @@ public interface IReviewManager public Task DeleteReviewAsync(ClaimsPrincipal user, string id); public Task GetReviewAsync(ClaimsPrincipal user, string id); public Task AddRevisionAsync(ClaimsPrincipal user, string reviewId, string name, string label, Stream fileStream, bool awaitComputeDiff = false); - public Task CreateCodeFile(string originalName, Stream fileStream, bool runAnalysis, MemoryStream memoryStream); + public Task CreateCodeFile(string originalName, Stream fileStream, bool runAnalysis, MemoryStream memoryStream, string language = null); public Task CreateReviewCodeFileModel(string revisionId, MemoryStream memoryStream, CodeFile codeFile); public Task DeleteRevisionAsync(ClaimsPrincipal user, string id, string revisionId); public Task UpdateRevisionLabelAsync(ClaimsPrincipal user, string id, string revisionId, string label); diff --git a/src/dotnet/APIView/APIViewWeb/Managers/ReviewManager.cs b/src/dotnet/APIView/APIViewWeb/Managers/ReviewManager.cs index c25677c8c9f..73be6495e5b 100644 --- a/src/dotnet/APIView/APIViewWeb/Managers/ReviewManager.cs +++ b/src/dotnet/APIView/APIViewWeb/Managers/ReviewManager.cs @@ -62,17 +62,17 @@ public ReviewManager ( _packageNameManager = packageNameManager; } - public async Task CreateReviewAsync(ClaimsPrincipal user, string originalName, string label, Stream fileStream, bool runAnalysis, bool awaitComputeDiff = false) + public async Task CreateReviewAsync(ClaimsPrincipal user, string originalName, string label, Stream fileStream, bool runAnalysis, string langauge, bool awaitComputeDiff = false) { var review = new ReviewModel { Author = user.GetGitHubLogin(), CreationDate = DateTime.UtcNow, RunAnalysis = runAnalysis, - Name = originalName, + Name = fileStream != null? originalName : Path.GetFileName(originalName), FilterType = ReviewType.Manual }; - await AddRevisionAsync(user, review, originalName, label, fileStream, awaitComputeDiff); + await AddRevisionAsync(user, review, originalName, label, fileStream, langauge, awaitComputeDiff); return review; } @@ -203,22 +203,27 @@ public async Task AddRevisionAsync( string reviewId, string name, string label, - Stream fileStream, bool awaitComputeDiff = false) + Stream fileStream, + bool awaitComputeDiff = false) { var review = await GetReviewAsync(user, reviewId); await AssertAutomaticReviewModifier(user, review); - await AddRevisionAsync(user, review, name, label, fileStream, awaitComputeDiff); + await AddRevisionAsync(user, review, name, label, fileStream, review.Language, awaitComputeDiff); } public async Task CreateCodeFile( string originalName, Stream fileStream, bool runAnalysis, - MemoryStream memoryStream) + MemoryStream memoryStream, + string language = null) { - var languageService = _languageServices.FirstOrDefault(s => s.IsSupportedFile(originalName)); - await fileStream.CopyToAsync(memoryStream); - memoryStream.Position = 0; + var languageService = _languageServices.FirstOrDefault(s => (language != null ? s.Name == language : s.IsSupportedFile(originalName))); + if (fileStream != null) + { + await fileStream.CopyToAsync(memoryStream); + memoryStream.Position = 0; + } CodeFile codeFile = null; if (languageService.IsReviewGenByPipeline) { @@ -242,10 +247,12 @@ public async Task CreateReviewCodeFileModel(string revision }; InitializeFromCodeFile(reviewCodeFileModel, codeFile); - memoryStream.Position = 0; - await _originalsRepository.UploadOriginalAsync(reviewCodeFileModel.ReviewFileId, memoryStream); + if (memoryStream != null) + { + memoryStream.Position = 0; + await _originalsRepository.UploadOriginalAsync(reviewCodeFileModel.ReviewFileId, memoryStream); + } await _codeFileRepository.UpsertCodeFileAsync(revisionId, reviewCodeFileModel.ReviewFileId, codeFile); - return reviewCodeFileModel; } @@ -698,6 +705,7 @@ private async Task AddRevisionAsync( string name, string label, Stream fileStream, + string language, bool awaitComputeDiff = false) { var revision = new ReviewRevisionModel(); @@ -706,12 +714,12 @@ private async Task AddRevisionAsync( revision.RevisionId, name, fileStream, - review.RunAnalysis); + review.RunAnalysis, + language); revision.Files.Add(codeFile); revision.Author = user.GetGitHubLogin(); revision.Label = label; - review.Revisions.Add(revision); if (review.PackageName != null) @@ -721,12 +729,12 @@ private async Task AddRevisionAsync( review.ServiceName = p?.ServiceName ?? review.ServiceName; } - var languageService = _languageServices.FirstOrDefault(s => s.IsSupportedFile(name)); - //Run pipeline to generate the review if sandbox is enabled + var languageService = language != null ? _languageServices.FirstOrDefault( l=> l.Name == language) : _languageServices.FirstOrDefault(s => s.IsSupportedFile(name)); + // Run pipeline to generate the review if sandbox is enabled if (languageService != null && languageService.IsReviewGenByPipeline) { // Run offline review gen for review and reviewCodeFileModel - await GenerateReviewOffline(review, revision.RevisionId, codeFile.ReviewFileId, name); + await GenerateReviewOffline(review, revision.RevisionId, codeFile.ReviewFileId, name, language); } // auto subscribe revision creation user @@ -747,10 +755,11 @@ private async Task CreateFileAsync( string revisionId, string originalName, Stream fileStream, - bool runAnalysis) + bool runAnalysis, + string language) { using var memoryStream = new MemoryStream(); - var codeFile = await CreateCodeFile(originalName, fileStream, runAnalysis, memoryStream); + var codeFile = await CreateCodeFile(originalName, fileStream, runAnalysis, memoryStream, language); var reviewCodeFileModel = await CreateReviewCodeFileModel(revisionId, memoryStream, codeFile); reviewCodeFileModel.FileName = originalName; return reviewCodeFileModel; @@ -934,8 +943,10 @@ private async Task FindMatchingApprovedRevision(ReviewModel return null; } - private async Task GenerateReviewOffline(ReviewModel review, string revisionId, string fileId, string fileName) + private async Task GenerateReviewOffline(ReviewModel review, string revisionId, string fileId, string fileName, string language = null) { + var languageService = _languageServices.Single(s => s.Name == language || s.Name == review.Language); + var revision = review.Revisions.FirstOrDefault(r => r.RevisionId == revisionId); var param = new ReviewGenPipelineParamModel() { FileID = fileId, @@ -943,9 +954,14 @@ private async Task GenerateReviewOffline(ReviewModel review, string revisionId, RevisionID = revisionId, FileName = fileName }; + if (!languageService.GeneratePipelineRunParams(param)) + { + throw new Exception($"Failed to run pipeline for review: {param.ReviewID}, file: {param.FileName}"); + } + var paramList = new List(); paramList.Add(param); - var languageService = _languageServices.Single(s => s.Name == review.Language); + await RunReviewGenPipeline(paramList, languageService.Name); } diff --git a/src/dotnet/APIView/APIViewWeb/Models/ReviewGenPipelineParamModel.cs b/src/dotnet/APIView/APIViewWeb/Models/ReviewGenPipelineParamModel.cs index aa07ae9b9f6..1efd454fec3 100644 --- a/src/dotnet/APIView/APIViewWeb/Models/ReviewGenPipelineParamModel.cs +++ b/src/dotnet/APIView/APIViewWeb/Models/ReviewGenPipelineParamModel.cs @@ -1,10 +1,12 @@ -namespace APIViewWeb.Models +namespace APIViewWeb.Models { public class ReviewGenPipelineParamModel { public string ReviewID { get; set; } public string RevisionID { get; set; } public string FileID { get; set; } - public string FileName { get; set; } + public string FileName { get; set; } + public string SourceRepoName { get; set; } + public string SourceBranchName { get; set; } } } diff --git a/src/dotnet/APIView/APIViewWeb/Models/UploadModel.cs b/src/dotnet/APIView/APIViewWeb/Models/UploadModel.cs index 82b34c924ab..51236dd079b 100644 --- a/src/dotnet/APIView/APIViewWeb/Models/UploadModel.cs +++ b/src/dotnet/APIView/APIViewWeb/Models/UploadModel.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using Microsoft.AspNetCore.Http; @@ -13,5 +13,13 @@ public class UploadModel [BindProperty] public IFormFile[] Files { get; set; } + + [BindProperty] + public string Language { get; set; } + + [BindProperty] + public string FilePath { get; set; } + + public static string[] SupportedLanguages => LanguageService.SupportedLanguages; } -} \ No newline at end of file +} diff --git a/src/dotnet/APIView/APIViewWeb/Pages/Assemblies/Index.cshtml b/src/dotnet/APIView/APIViewWeb/Pages/Assemblies/Index.cshtml index 7277a411de3..4290ab17474 100644 --- a/src/dotnet/APIView/APIViewWeb/Pages/Assemblies/Index.cshtml +++ b/src/dotnet/APIView/APIViewWeb/Pages/Assemblies/Index.cshtml @@ -57,16 +57,30 @@
-
+ + + +
+
+
+
+ +
-
- -