Server changes to create manual and upgrade required API reviews usin…

…g pipeline(sandboxing) (#3774)

* Generate reviews using sandboxing pipeline

src/dotnet/APIView/APIViewWeb/APIViewWeb.csproj

@@ -27,6 +27,12 @@
     <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.12.0-beta4" />
     <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.26.1" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation" Version="3.0" />
+    <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.170.0" />
+    <PackageReference Include="Microsoft.TypeScript.MSBuild" Version="4.7.2">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="16.170.0" />
     <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="2.1.9" PrivateAssets="All" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0-beta2-19367-01" PrivateAssets="All" />
     <PackageReference Include="Markdig.Signed" Version="0.17.1" />

src/dotnet/APIView/APIViewWeb/Controllers/ReviewController.cs

@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using APIViewWeb.Filters;
+using APIViewWeb.Repositories;
+using Microsoft.ApplicationInsights;
+using Microsoft.ApplicationInsights.DataContracts;
+using Microsoft.ApplicationInsights.Extensibility;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace APIViewWeb.Controllers
+{
+    public class ReviewController : Controller
+    {
+        private readonly ReviewManager _reviewManager;
+        private readonly ILogger _logger;
+
+        public ReviewController(ReviewManager reviewManager, ILogger<ReviewController> logger)
+        {
+            _reviewManager = reviewManager;
+            _logger = logger;
+        }
+
+        [HttpGet]
+        public async Task<ActionResult> UpdateApiReview(string repoName, string artifactPath, string buildId, string project = "internal")
+        {
+            await _reviewManager.UpdateReviewCodeFiles(repoName, buildId, artifactPath, project);
+            return Ok();
+        }
+    }
+}

src/dotnet/APIView/APIViewWeb/Languages/LanguageProcessor.cs

@@ -1,9 +1,12 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Text.Json;
 using System.Threading.Tasks;
 using ApiView;
-
+using APIViewWeb.Models;
+
 namespace APIViewWeb
 {
     public abstract class LanguageProcessor: LanguageService

src/dotnet/APIView/APIViewWeb/Languages/LanguageService.cs

@@ -1,7 +1,10 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Threading.Tasks;
 using ApiView;
+using APIView;
+using APIViewWeb.Models;
 
 namespace APIViewWeb
 {
@@ -12,5 +15,16 @@ public abstract class LanguageService
         public virtual bool IsSupportedFile(string name) => name.EndsWith(Extension, StringComparison.OrdinalIgnoreCase);
         public abstract bool CanUpdate(string versionString);
         public abstract Task<CodeFile> GetCodeFileAsync(string originalName, Stream stream, bool runAnalysis);
+        public virtual bool IsReviewGenByPipeline { get; } = false;
+
+        public readonly CodeFileToken ReviewNotReadyCodeFile = new CodeFileToken("API review is being generated for this revision and it will be available in few minutes. Please refresh this page after few minutes to see generated API review.", CodeFileTokenKind.Literal);
+        public virtual CodeFile GetReviewGenPendingCodeFile(string fileName) => new CodeFile()
+        {
+            Name = fileName,
+            PackageName = fileName,
+            Language = Name,
+            Tokens = new CodeFileToken[] {new CodeFileToken("", CodeFileTokenKind.Newline), ReviewNotReadyCodeFile, new CodeFileToken("", CodeFileTokenKind.Newline) },
+            Navigation = new NavigationItem[] { new NavigationItem() { Text = fileName } }
+        };
     }
 }

src/dotnet/APIView/APIViewWeb/Models/ReviewGenPipelineParamModel.cs

@@ -0,0 +1,10 @@
+namespace APIViewWeb.Models
+{
+    public class ReviewGenPipelineParamModel
+    {
+        public string ReviewID { get; set; }
+        public string RevisionID { get; set; }
+        public string FileID { get; set; }
+        public string FileName { get; set; }
+    }
+}

src/dotnet/APIView/APIViewWeb/Pages/Assemblies/Review.cshtml.cs

@@ -221,13 +221,6 @@ private int ComputeActiveConversations(CodeLine[] lines, ReviewCommentsModel com
             return activeThreads;
         }
 
-        public async Task<ActionResult> OnPostRefreshModelAsync(string id)
-        {
-            await _manager.UpdateReviewAsync(User, id);
-
-            return RedirectToPage(new { id = id });
-        }
-
         public async Task<ActionResult> OnPostToggleClosedAsync(string id)
         {
             await _manager.ToggleIsClosedAsync(User, id);

src/dotnet/APIView/APIViewWeb/Repositories/BlobOriginalsRepository.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System;
 using System.IO;
 using System.Threading.Tasks;
 using Azure.Storage.Blobs;
@@ -12,6 +13,8 @@ public class BlobOriginalsRepository
     {
         private BlobContainerClient _container;
 
+        public string GetContainerUrl() => _container.Uri.ToString();
+
         public BlobOriginalsRepository(IConfiguration configuration)
         {
             var connectionString = configuration["Blob:ConnectionString"];
@@ -39,4 +42,4 @@ public async Task DeleteOriginalAsync(string codeFileId)
             await GetBlobClient(codeFileId).DeleteAsync();
         }
     }
-}
+}

src/dotnet/APIView/APIViewWeb/Repositories/DevopsArtifactRepository.cs

@@ -1,10 +1,11 @@
-using Microsoft.Extensions.Configuration;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Configuration;
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Headers;
+using System.Text;
 using System.Text.Json;
 using System.Threading.Tasks;
 
@@ -16,23 +17,31 @@ public class DevopsArtifactRepository
         private readonly IConfiguration _configuration;
 
         private readonly string _devopsAccessToken;
+        private readonly string _pipeline_run_rest;
+        private readonly string _hostUrl;
 
-        public DevopsArtifactRepository(IConfiguration configuration)
+        public DevopsArtifactRepository(IConfiguration configuration, IHttpContextAccessor httpContextAccessor)
         {
             _configuration = configuration;
             _devopsAccessToken = Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", _configuration["Azure-Devops-PAT"])));
+            _pipeline_run_rest = _configuration["Azure-Devops-Run-Ripeline-Rest"];
+            _hostUrl = _configuration["APIVIew-Host-Url"];
         }
 
         public async Task<Stream> DownloadPackageArtifact(string repoName, string buildId, string artifactName, string filePath, string project, string format= "file")
         {
             var downloadUrl = await GetDownloadArtifactUrl(repoName, buildId, artifactName, project);
             if (!string.IsNullOrEmpty(downloadUrl))
             {
-                if (!filePath.StartsWith("/"))
+                if(!string.IsNullOrEmpty(filePath))
                 {
-                    filePath = "/" + filePath;
+                    if (!filePath.StartsWith("/"))
+                    {
+                        filePath = "/" + filePath;
+                    }
+                    downloadUrl = downloadUrl.Split("?")[0] + "?format=" + format + "&subPath=" + filePath;
                 }
-                downloadUrl = downloadUrl.Split("?")[0] + "?format=" + format + "&subPath=" + filePath;
+
                 SetDevopsClientHeaders();
                 var downloadResp = await _devopsClient.GetAsync(downloadUrl);
                 downloadResp.EnsureSuccessStatusCode();
@@ -71,5 +80,16 @@ private string GetArtifactRestAPIForRepo(string repoName)
             }
             return downloadArtifactRestApi;
         }
+
+        public async Task RunPipeline(string pipelineName, string reviewDetails, string originalStorageUrl)
+        {
+            SetDevopsClientHeaders();
+            //Create dictionary of all required parametes to run tools - generate-<language>-apireview pipeline in azure devops
+            var reviewDetailsDict = new Dictionary<string, string> { { "Reviews", reviewDetails }, { "APIViewUrl", _hostUrl }, { "StorageContainerUrl", originalStorageUrl } };
+            var pipelineParams = new Dictionary<string, Dictionary<string, string>> { { "templateParameters", reviewDetailsDict } };
+            var stringContent = new StringContent(JsonSerializer.Serialize(pipelineParams), Encoding.UTF8, "application/json");
+            var response = await _devopsClient.PostAsync(_pipeline_run_rest, stringContent);
+            response.EnsureSuccessStatusCode();
+        }
     }
 }

src/dotnet/APIView/APIViewWeb/Repositories/ReviewManager.cs

@@ -17,6 +17,7 @@
 using Microsoft.ApplicationInsights.DataContracts;
 using Microsoft.ApplicationInsights.Extensibility;
 using Microsoft.AspNetCore.Authorization;
+using Microsoft.Extensions.Configuration;
 
 namespace APIViewWeb.Repositories
 {
@@ -51,8 +52,9 @@ public ReviewManager(
             CosmosCommentsRepository commentsRepository,
             IEnumerable<LanguageService> languageServices,
             NotificationManager notificationManager,
-            DevopsArtifactRepository devopsArtifactRepository,
-            PackageNameManager packageNameManager)
+            DevopsArtifactRepository devopsClient,
+            PackageNameManager packageNameManager,
+            IConfiguration configuration)
         {
             _authorizationService = authorizationService;
             _reviewsRepository = reviewsRepository;
@@ -61,7 +63,7 @@ public ReviewManager(
             _commentsRepository = commentsRepository;
             _languageServices = languageServices;
             _notificationManager = notificationManager;
-            _devopsArtifactRepository = devopsArtifactRepository;
+            _devopsArtifactRepository = devopsClient;
             _packageNameManager = packageNameManager;
         }
 
@@ -188,31 +190,32 @@ private async Task UpdateReviewAsync(ReviewModel review)
                         var fileOriginal = await _originalsRepository.GetOriginalAsync(file.ReviewFileId);
                         var languageService = GetLanguageService(file.Language);
                         if (languageService == null)
-                            continue;
+                            continue;
 
                         // file.Name property has been repurposed to store package name and version string
                         // This is causing issue when updating review using latest parser since it expects Name field as file name
                         // We have added a new property FileName which is only set for new reviews
                         // All older reviews needs to be handled by checking review name field
                         var fileName = file.FileName ?? (Path.HasExtension(review.Name) ? review.Name : file.Name);
-                        var codeFile = await languageService.GetCodeFileAsync(fileName, fileOriginal, review.RunAnalysis);
-                        await _codeFileRepository.UpsertCodeFileAsync(revision.RevisionId, file.ReviewFileId, codeFile);
-                        // update only version string
-                        file.VersionString = codeFile.VersionString;
+                        if (languageService.IsReviewGenByPipeline)
+                        {
+                            GenerateReviewOffline(review, revision.RevisionId, file.ReviewFileId, fileName);
+                        }
+                        else
+                        {
+                            var codeFile = await languageService.GetCodeFileAsync(fileName, fileOriginal, review.RunAnalysis);
+                            await _codeFileRepository.UpsertCodeFileAsync(revision.RevisionId, file.ReviewFileId, codeFile);
+                            // update only version string
+                            file.VersionString = codeFile.VersionString;
+                            await _reviewsRepository.UpsertReviewAsync(review);
+                        }                        
                     }
                     catch (Exception ex) {
                         _telemetryClient.TrackTrace("Failed to update review " + review.ReviewId);
                         _telemetryClient.TrackException(ex);
                     }                    
                 }
-            }
-            await _reviewsRepository.UpsertReviewAsync(review);
-        }
-
-        internal async Task UpdateReviewAsync(ClaimsPrincipal user, string id)
-        {
-            var review = await GetReviewAsync(user, id);
-            await UpdateReviewAsync(review);
+            }            
         }
 
         public async Task AddRevisionAsync(
@@ -255,9 +258,16 @@ private async Task AddRevisionAsync(
                 review.ServiceName = p?.ServiceName ?? review.ServiceName;
             }
 
+            var languageService = _languageServices.Single(s => s.IsSupportedFile(name));
+            //Run pipeline to generateteh review if sandbox is enabled
+            if (languageService.IsReviewGenByPipeline)
+            {
+                // Run offline review gen for review and reviewCodeFileModel
+                GenerateReviewOffline(review, revision.RevisionId, codeFile.ReviewFileId, name);
+            }
+
             // auto subscribe revision creation user
             await _notificationManager.SubscribeAsync(review, user);
-
             await _reviewsRepository.UpsertReviewAsync(review);
             await _notificationManager.NotifySubscribersOnNewRevisionAsync(revision, user);
         }
@@ -271,7 +281,7 @@ private async Task<ReviewCodeFileModel> CreateFileAsync(
             using var memoryStream = new MemoryStream();
             var codeFile = await CreateCodeFile(originalName, fileStream, runAnalysis, memoryStream);
             var reviewCodeFileModel = await CreateReviewCodeFileModel(revisionId, memoryStream, codeFile);
-            reviewCodeFileModel.FileName = originalName;
+            reviewCodeFileModel.FileName = originalName;            
             return reviewCodeFileModel;
         }
 
@@ -284,12 +294,18 @@ public async Task<CodeFile> CreateCodeFile(
             var languageService = _languageServices.FirstOrDefault(s => s.IsSupportedFile(originalName));
             await fileStream.CopyToAsync(memoryStream);
             memoryStream.Position = 0;
-
-            CodeFile codeFile = await languageService.GetCodeFileAsync(
+            CodeFile codeFile = null;
+            if (languageService.IsReviewGenByPipeline)
+            {
+                codeFile = languageService.GetReviewGenPendingCodeFile(originalName);
+            }
+            else
+            {
+                codeFile = await languageService.GetCodeFileAsync(
                 originalName,
                 memoryStream,
                 runAnalysis);
-
+            }
             return codeFile;
         }
 
@@ -696,5 +712,63 @@ public async Task AutoArchiveReviews(int archiveAfterMonths)
                 }
             }
         }
+        private void GenerateReviewOffline(ReviewModel review, string revisionId, string fileId, string fileName)
+        {
+            var param = new ReviewGenPipelineParamModel()
+            {
+                FileID = fileId,
+                ReviewID = review.ReviewId,
+                RevisionID = revisionId,
+                FileName = fileName
+            };
+            var paramList = new List<ReviewGenPipelineParamModel>();
+            paramList.Add(param);
+            var languageService = _languageServices.Single(s => s.Name == review.Language);
+            RunReviewGenPipeline(paramList, languageService.Name);
+        }
+
+        public async Task UpdateReviewCodeFiles(string repoName, string buildId, string artifact, string project)
+        {
+            var stream = await _devopsArtifactRepository.DownloadPackageArtifact(repoName, buildId, artifact, filePath: null, project: project, format: "zip");
+            var archive = new ZipArchive(stream);
+            foreach (var entry in archive.Entries)
+            {
+                var reviewFilePath = entry.FullName;
+                var reviewDetails = reviewFilePath.Split("/");
+
+                if (reviewDetails.Length < 4 || !reviewFilePath.EndsWith(".json"))
+                    continue;
+
+                var reviewId = reviewDetails[1];
+                var revisionId = reviewDetails[2];
+                var codeFile = await CodeFile.DeserializeAsync(entry.Open());
+
+                // Update code file with one downloaded from pipeline
+                var review = await _reviewsRepository.GetReviewAsync(reviewId);
+                if (review != null)
+                {
+                    var revision = review.Revisions.SingleOrDefault(review => review.RevisionId == revisionId);
+                    if (revision != null)
+                    {
+                        await _codeFileRepository.UpsertCodeFileAsync(revisionId, revision.SingleFile.ReviewFileId, codeFile);
+                        revision.Files.FirstOrDefault().VersionString = codeFile.VersionString;
+                        await _reviewsRepository.UpsertReviewAsync(review);
+                    }
+                }
+            }
+        }
+        private async void RunReviewGenPipeline(List<ReviewGenPipelineParamModel> reviewGenParams, string language)
+        {
+            var jsonSerializerOptions = new JsonSerializerOptions()
+            {
+                AllowTrailingCommas = true,
+                ReadCommentHandling = JsonCommentHandling.Skip
+            };
+            var reviewParamString = JsonSerializer.Serialize(reviewGenParams, jsonSerializerOptions);
+            reviewParamString = reviewParamString.Replace("\"", "'");
+            await _devopsArtifactRepository.RunPipeline($"tools - generate-{language}-apireview", 
+                reviewParamString, 
+                _originalsRepository.GetContainerUrl());
+        }
     }
 }