Add e2e test for custom analyzer rules

src/Build.UnitTests/Evaluation/Expander_Tests.cs

@@ -5071,7 +5071,7 @@ private static bool ICUModeAvailable()
         }
 
         [Fact]
-        public void PropertyFunctionRegisterAnalyzer()
+        public void PropertyFunctionRegisterBuildCheck()
         {
             using (var env = TestEnvironment.Create())
             {
@@ -5084,7 +5084,7 @@ public void PropertyFunctionRegisterAnalyzer()
                 var dummyAssemblyFile = env.CreateFile(env.CreateFolder(), "test.dll");
 
                 var result = new Expander<ProjectPropertyInstance, ProjectItemInstance>(new PropertyDictionary<ProjectPropertyInstance>(), FileSystems.Default)
-                    .ExpandIntoStringLeaveEscaped($"$([MSBuild]::RegisterAnalyzer({dummyAssemblyFile.Path}))", ExpanderOptions.ExpandProperties, MockElementLocation.Instance, loggingContext);
+                    .ExpandIntoStringLeaveEscaped($"$([MSBuild]::RegisterBuildCheck({dummyAssemblyFile.Path}))", ExpanderOptions.ExpandProperties, MockElementLocation.Instance, loggingContext);
 
                 result.ShouldBe(Boolean.TrueString);
                 _ = logger.AllBuildEvents.Select(be => be.ShouldBeOfType<BuildCheckAcquisitionEventArgs>());

src/Build/BuildCheck/Infrastructure/BuildCheckConnectorLogger.cs

@@ -9,7 +9,6 @@
 using Microsoft.Build.BuildCheck.Utilities;
 using Microsoft.Build.Experimental.BuildCheck;
 using Microsoft.Build.Framework;
-using static Microsoft.Build.BuildCheck.Infrastructure.BuildCheckManagerProvider;
 
 namespace Microsoft.Build.BuildCheck.Infrastructure;
 
@@ -67,6 +66,14 @@ private void HandleProjectEvaluationStartedEvent(ProjectEvaluationStartedEventAr
         }
     }
 
+    private void HandleBuildCheckTracingEvent(BuildCheckTracingEventArgs eventArgs)
+    {
+        if (!eventArgs.IsAggregatedGlobalReport)
+        {
+            _stats.Merge(eventArgs.TracingData, (span1, span2) => span1 + span2);
+        }
+    }
+
     private bool IsMetaProjFile(string? projectFile) => !string.IsNullOrEmpty(projectFile) && projectFile!.EndsWith(".metaproj", StringComparison.OrdinalIgnoreCase);
 
     private void EventSource_AnyEventRaised(object sender, BuildEventArgs e)
@@ -81,19 +88,12 @@ private void EventSource_AnyEventRaised(object sender, BuildEventArgs e)
 
     private void EventSource_BuildFinished(object sender, BuildFinishedEventArgs e)
     {
-        BuildEventContext buildEventContext = e.BuildEventContext
-            ?? new BuildEventContext(
-                BuildEventContext.InvalidNodeId,
-                BuildEventContext.InvalidTargetId,
-                BuildEventContext.InvalidProjectContextId,
-                BuildEventContext.InvalidTaskId);
-
-        LoggingContext loggingContext = _loggingContextFactory.CreateLoggingContext(buildEventContext);
+        LoggingContext loggingContext = _loggingContextFactory.CreateLoggingContext(GetBuildEventContext(e));
 
         _stats.Merge(_buildCheckManager.CreateAnalyzerTracingStats(), (span1, span2) => span1 + span2);
         LogAnalyzerStats(loggingContext);
     }
-    
+
     private void LogAnalyzerStats(LoggingContext loggingContext)
     {
         Dictionary<string, TimeSpan> infraStats = new Dictionary<string, TimeSpan>();
@@ -131,18 +131,18 @@ private string BuildCsvString(string title, Dictionary<string, TimeSpan> rowData
 
     private Dictionary<Type, Action<BuildEventArgs>> GetBuildEventHandlers() => new()
     {
-        { typeof(ProjectEvaluationFinishedEventArgs), (BuildEventArgs e) => HandleProjectEvaluationFinishedEvent((ProjectEvaluationFinishedEventArgs) e) },
-        { typeof(ProjectEvaluationStartedEventArgs), (BuildEventArgs e) => HandleProjectEvaluationStartedEvent((ProjectEvaluationStartedEventArgs) e) },
+        { typeof(ProjectEvaluationFinishedEventArgs), (BuildEventArgs e) => HandleProjectEvaluationFinishedEvent((ProjectEvaluationFinishedEventArgs)e) },
+        { typeof(ProjectEvaluationStartedEventArgs), (BuildEventArgs e) => HandleProjectEvaluationStartedEvent((ProjectEvaluationStartedEventArgs)e) },
         { typeof(ProjectStartedEventArgs), (BuildEventArgs e) => _buildCheckManager.StartProjectRequest(BuildCheckDataSource.EventArgs, e.BuildEventContext!) },
         { typeof(ProjectFinishedEventArgs), (BuildEventArgs e) => _buildCheckManager.EndProjectRequest(BuildCheckDataSource.EventArgs, e.BuildEventContext!) },
-        { typeof(BuildCheckTracingEventArgs), (BuildEventArgs e) => 
-        {
-            if(!((BuildCheckTracingEventArgs)e).IsAggregatedGlobalReport)
-            {
-                _stats.Merge(((BuildCheckTracingEventArgs)e).TracingData, (span1, span2) => span1 + span2);
-            }
-        }
-        },
-        { typeof(BuildCheckAcquisitionEventArgs), (BuildEventArgs e) => _buildCheckManager.ProcessAnalyzerAcquisition(((BuildCheckAcquisitionEventArgs)e).ToAnalyzerAcquisitionData(), e.BuildEventContext!) },
+        { typeof(BuildCheckTracingEventArgs), (BuildEventArgs e) => HandleBuildCheckTracingEvent((BuildCheckTracingEventArgs)e) },
+        { typeof(BuildCheckAcquisitionEventArgs), (BuildEventArgs e) => _buildCheckManager.ProcessAnalyzerAcquisition(((BuildCheckAcquisitionEventArgs)e).ToAnalyzerAcquisitionData(), GetBuildEventContext(e)) },
     };
+
+    private BuildEventContext GetBuildEventContext(BuildEventArgs e) => e.BuildEventContext
+        ?? new BuildEventContext(
+                BuildEventContext.InvalidNodeId,
+                BuildEventContext.InvalidTargetId,
+                BuildEventContext.InvalidProjectContextId,
+                BuildEventContext.InvalidTaskId);
 }

src/Build/Evaluation/Expander.cs

@@ -3917,12 +3917,12 @@ private bool TryExecuteWellKnownFunction(out object returnVal, object objectInst
                     }
                     else if (_receiverType == typeof(IntrinsicFunctions))
                     {
-                        if (string.Equals(_methodMethodName, nameof(IntrinsicFunctions.RegisterAnalyzer), StringComparison.OrdinalIgnoreCase))
+                        if (string.Equals(_methodMethodName, nameof(IntrinsicFunctions.RegisterBuildCheck), StringComparison.OrdinalIgnoreCase))
                         {
-                            ErrorUtilities.VerifyThrow(_loggingContext != null, $"The logging context is missed. {nameof(IntrinsicFunctions.RegisterAnalyzer)} can not be invoked.");
+                            ErrorUtilities.VerifyThrow(_loggingContext != null, $"The logging context is missed. {nameof(IntrinsicFunctions.RegisterBuildCheck)} can not be invoked.");
                             if (TryGetArg(args, out string arg0))
                             {
-                                returnVal = IntrinsicFunctions.RegisterAnalyzer(arg0, _loggingContext);
+                                returnVal = IntrinsicFunctions.RegisterBuildCheck(arg0, _loggingContext);
                                 return true;
                             }
                         }

src/Build/Evaluation/IntrinsicFunctions.cs

@@ -697,7 +697,7 @@ public static string GetMSBuildExtensionsPath()
 
         public static bool IsRunningFromVisualStudio() => BuildEnvironmentHelper.Instance.Mode == BuildEnvironmentMode.VisualStudio;
 
-        public static bool RegisterAnalyzer(string pathToAssembly, LoggingContext loggingContext)
+        public static bool RegisterBuildCheck(string pathToAssembly, LoggingContext loggingContext)
         {
             pathToAssembly = FileUtilities.GetFullPathNoThrow(pathToAssembly);
             if (File.Exists(pathToAssembly))

src/BuildCheck.UnitTests/EndToEndTests.cs

@@ -4,12 +4,10 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
-using System.Reflection;
-using System.Text;
-using System.Threading.Tasks;
+using System.Xml;
 using Microsoft.Build.UnitTests;
 using Microsoft.Build.UnitTests.Shared;
+using Newtonsoft.Json.Linq;
 using Shouldly;
 using Xunit;
 using Xunit.Abstractions;
@@ -19,6 +17,7 @@ namespace Microsoft.Build.BuildCheck.UnitTests;
 public class EndToEndTests : IDisposable
 {
     private readonly TestEnvironment _env;
+
     public EndToEndTests(ITestOutputHelper output)
     {
         _env = TestEnvironment.Create(output);
@@ -27,6 +26,8 @@ public EndToEndTests(ITestOutputHelper output)
         _env.WithEnvironmentInvariant();
     }
 
+    private static string TestAssetsRootPath { get; } = Path.Combine(Path.GetDirectoryName(typeof(EndToEndTests).Assembly.Location) ?? AppContext.BaseDirectory, "TestAssets");
+
     public void Dispose() => _env.Dispose();
 
     [Theory]
@@ -91,7 +92,6 @@ public void SampleAnalyzerIntegrationTest(bool buildInOutOfProcessNode, bool ana
         // var cache = new SimpleProjectRootElementCache();
         // ProjectRootElement xml = ProjectRootElement.OpenProjectOrSolution(projectFile.Path, /*unused*/null, /*unused*/null, cache, false /*Not explicitly loaded - unused*/);
 
-
         TransientTestFile config = _env.CreateFile(workFolder, "editorconfig.json",
             /*lang=json,strict*/
             """
@@ -134,4 +134,90 @@ public void SampleAnalyzerIntegrationTest(bool buildInOutOfProcessNode, bool ana
             output.ShouldNotContain("BC0101");
         }
     }
+
+    [Theory]
+    [InlineData(new[] { "CustomAnalyzer" }, "AnalysisCandidate", new[] { "CustomRule1", "CustomRule2" })]
+    [InlineData(new[] { "CustomAnalyzer", "CustomAnalyzer2" }, "AnalysisCandidateWithMultipleAnalyzersInjected", new[] { "CustomRule1", "CustomRule2", "CustomRule3" })]
+    public void CustomAnalyzerTest(string[] customAnalyzerNames, string analysisCandidate, string[] expectedRegisteredRules)
+    {
+        using (var env = TestEnvironment.Create())
+        {
+            var candidatesNugetFullPaths = BuildAnalyzerRules(env, customAnalyzerNames);
+
+            candidatesNugetFullPaths.ShouldNotBeEmpty("Nuget package with custom analyzer was not generated or detected.");
+
+            var analysisCandidatePath = Path.Combine(TestAssetsRootPath, analysisCandidate);
+            AddCustomDataSourceToNugetConfig(analysisCandidatePath, candidatesNugetFullPaths);
+
+            string projectAnalysisBuildLog = RunnerUtilities.ExecBootstrapedMSBuild(
+                $"{Path.Combine(analysisCandidatePath, $"{analysisCandidate}.csproj")} /m:1 -nr:False -restore /p:OutputPath={env.CreateFolder().Path} -analyze -verbosity:d",
+                out bool successBuild);
+            successBuild.ShouldBeTrue();
+
+            foreach (string expectedRegisteredRule in expectedRegisteredRules)
+            {
+                projectAnalysisBuildLog.ShouldContain($"Custom analyzer rule: {expectedRegisteredRule} has been registered successfully.");
+            }
+        }
+    }
+
+    private IList<string> BuildAnalyzerRules(TestEnvironment env, string[] customAnalyzerNames)
+    {
+        var candidatesNugetFullPaths = new List<string>();
+
+        foreach (var customAnalyzerName in customAnalyzerNames)
+        {
+            var candidateAnalysisProjectPath = Path.Combine(TestAssetsRootPath, customAnalyzerName, $"{customAnalyzerName}.csproj");
+            var nugetPackResults = RunnerUtilities.ExecBootstrapedMSBuild(
+                 $"{candidateAnalysisProjectPath} /m:1 -nr:False -restore /p:OutputPath={env.CreateFolder().Path} -getTargetResult:Build", out bool success, attachProcessId: false);
+
+            success.ShouldBeTrue();
+
+            string? candidatesNugetPackageFullPath = (string?)(JObject.Parse(nugetPackResults)?["TargetResults"]?["Build"]?["Items"]?[0]?["RelativeDir"] ?? string.Empty);
+
+            candidatesNugetPackageFullPath.ShouldNotBeNull();
+            candidatesNugetFullPaths.Add(candidatesNugetPackageFullPath);
+        }
+
+        return candidatesNugetFullPaths;
+    }
+
+    private void AddCustomDataSourceToNugetConfig(string analysisCandidatePath, IList<string> candidatesNugetPackageFullPaths)
+    {
+        var nugetTemplatePath = Path.Combine(analysisCandidatePath, "nugetTemplate.config");
+
+        var doc = new XmlDocument();
+        doc.LoadXml(File.ReadAllText(nugetTemplatePath));
+        if (doc.DocumentElement != null)
+        {
+            XmlNode? packageSourcesNode = doc.SelectSingleNode("//packageSources");
+            for (int i = 0; i < candidatesNugetPackageFullPaths.Count; i++)
+            {
+                AddPackageSource(doc, packageSourcesNode, $"Key{i}", Path.GetDirectoryName(candidatesNugetPackageFullPaths[i]) ?? string.Empty);
+            }
+
+            doc.Save(Path.Combine(analysisCandidatePath, "nuget.config"));
+        }
+    }
+
+    private void AddPackageSource(XmlDocument doc, XmlNode? packageSourcesNode, string key, string value)
+    {
+        if (packageSourcesNode != null)
+        {
+            XmlElement addNode = doc.CreateElement("add");
+
+            PopulateXmlAttribute(doc, addNode, "key", key);
+            PopulateXmlAttribute(doc, addNode, "value", value);
+
+            packageSourcesNode.AppendChild(addNode);
+        }
+    }
+
+    private void PopulateXmlAttribute(XmlDocument doc, XmlNode node, string attributeName, string attributeValue)
+    {
+        node.ShouldNotBeNull($"The attribute {attributeName} can not be populated with {attributeValue}. Xml node is null.");
+        var attribute = doc.CreateAttribute(attributeName);
+        attribute.Value = attributeValue;
+        node.Attributes!.Append(attribute);
+    }
 }

src/BuildCheck.UnitTests/Microsoft.Build.BuildCheck.UnitTests.csproj

@@ -33,5 +33,9 @@
     <None Include="..\Shared\UnitTests\xunit.runner.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Include="TestAssets\**\*">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
+
 </Project>

src/BuildCheck.UnitTests/TestAssets/AnalysisCandidate/AnalysisCandidate.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CustomAnalyzer" Version="1.0.0"/>
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Include="nuget.config">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

src/BuildCheck.UnitTests/TestAssets/AnalysisCandidate/nugetTemplate.config

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+   <packageSources>
+
+  </packageSources>
+</configuration>

src/BuildCheck.UnitTests/TestAssets/AnalysisCandidateWithMultipleAnalyzersInjected/AnalysisCandidateWithMultipleAnalyzersInjected.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CustomAnalyzer" Version="1.0.0"/>
+    <PackageReference Include="CustomAnalyzer2" Version="1.0.0"/>
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Include="nuget.config">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

src/BuildCheck.UnitTests/TestAssets/AnalysisCandidateWithMultipleAnalyzersInjected/nugetTemplate.config

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+   <packageSources>
+
+  </packageSources>
+</configuration>

src/BuildCheck.UnitTests/TestAssets/CustomAnalyzer/Analyzer1.cs

@@ -0,0 +1,38 @@
+using System.Collections.Generic;
+using Microsoft.Build.Construction;
+using Microsoft.Build.Experimental.BuildCheck;
+
+namespace CustomAnalyzer
+{
+    public sealed class Analyzer1 : BuildAnalyzer
+    {
+        public static BuildAnalyzerRule SupportedRule = new BuildAnalyzerRule(
+            "X01234",
+            "Title",
+            "Description",
+            "Message format: {0}",
+            new BuildAnalyzerConfiguration());
+
+        public override string FriendlyName => "CustomRule1";
+
+        public override IReadOnlyList<BuildAnalyzerRule> SupportedRules { get; } = new List<BuildAnalyzerRule>() { SupportedRule };
+
+        public override void Initialize(ConfigurationContext configurationContext)
+        {
+            // configurationContext to be used only if analyzer needs external configuration data.
+        }
+
+        public override void RegisterActions(IBuildCheckRegistrationContext registrationContext)
+        {
+            registrationContext.RegisterEvaluatedPropertiesAction(EvaluatedPropertiesAction);
+        }
+
+        private void EvaluatedPropertiesAction(BuildCheckDataContext<EvaluatedPropertiesAnalysisData> context)
+        {
+            context.ReportResult(BuildCheckResult.Create(
+                SupportedRule,
+                ElementLocation.EmptyLocation,
+                "Argument for the message format"));
+        }
+    }
+}

src/BuildCheck.UnitTests/TestAssets/CustomAnalyzer/Analyzer2.cs

@@ -0,0 +1,38 @@
+using System.Collections.Generic;
+using Microsoft.Build.Construction;
+using Microsoft.Build.Experimental.BuildCheck;
+
+namespace CustomAnalyzer
+{
+    public sealed class Analyzer2 : BuildAnalyzer
+    {
+        public static BuildAnalyzerRule SupportedRule = new BuildAnalyzerRule(
+            "X01235",
+            "Title",
+            "Description",
+            "Message format: {0}",
+            new BuildAnalyzerConfiguration());
+
+        public override string FriendlyName => "CustomRule2";
+
+        public override IReadOnlyList<BuildAnalyzerRule> SupportedRules { get; } = new List<BuildAnalyzerRule>() { SupportedRule };
+
+        public override void Initialize(ConfigurationContext configurationContext)
+        {
+            // configurationContext to be used only if analyzer needs external configuration data.
+        }
+
+        public override void RegisterActions(IBuildCheckRegistrationContext registrationContext)
+        {
+            registrationContext.RegisterEvaluatedPropertiesAction(EvaluatedPropertiesAction);
+        }
+
+        private void EvaluatedPropertiesAction(BuildCheckDataContext<EvaluatedPropertiesAnalysisData> context)
+        {
+            context.ReportResult(BuildCheckResult.Create(
+                SupportedRule,
+                ElementLocation.EmptyLocation,
+                "Argument for the message format"));
+        }
+    }
+}