Add project template build tests + CG reporting

eng/build.proj

@@ -1,6 +1,7 @@
 <Project Sdk="Microsoft.Build.Traversal">
   <ItemGroup>
     <_SnapshotsToExclude Include="$(MSBuildThisFileDirectory)..\test\**\Snapshots\**\*.*proj" />
+    <_GeneratedContentToExclude Include="$(MSBuildThisFileDirectory)..\test\**\TemplateSandbox\**\*.*proj" />
 
     <!-- We recursively add all of the projects inside the src directory, except for the exclusions above -->
     <_ProjectsToBuild Include="$(MSBuildThisFileDirectory)..\src\**\*.csproj" />
@@ -11,6 +12,6 @@
     <_ProjectsToBuild Include="$(MSBuildThisFileDirectory)..\src\Packages\Microsoft.Internal.Extensions.DotNetApiDocs.Transport\Microsoft.Internal.Extensions.DotNetApiDocs.Transport.proj" />
 
     <!-- Add all the projects we want to build as project references, so the traversal SDK can build them -->
-    <ProjectReference Include="@(_ProjectsToBuild)" Exclude="@(_ProjectsToExclude);@(_SnapshotsToExclude)" />
+    <ProjectReference Include="@(_ProjectsToBuild)" Exclude="@(_ProjectsToExclude);@(_SnapshotsToExclude);@(_GeneratedContentToExclude)" />
   </ItemGroup>
-</Project>
+</Project>

eng/pipelines/templates/BuildAndTest.yml

@@ -56,7 +56,25 @@ steps:
               --settings $(Build.SourcesDirectory)/eng/CodeCoverage.config
               --output ${{ parameters.repoTestResultsPath }}/$(Agent.JobName)_CodeCoverageResults/$(Agent.JobName)_cobertura.xml
               "${{ parameters.buildScript }} -test -configuration ${{ parameters.buildConfig }} /bl:${{ parameters.repoLogPath }}/tests.binlog $(_OfficialBuildIdArgs)"
-      displayName: Run tests
+      displayName: Run unit tests
+
+  - script: ${{ parameters.buildScript }}
+            -pack
+            -configuration ${{ parameters.buildConfig }}
+            -warnAsError 1
+            /bl:${{ parameters.repoLogPath }}/pack.binlog
+            /p:Restore=false /p:Build=false
+            $(_OfficialBuildIdArgs)
+    displayName: Pack
+
+  - ${{ if ne(parameters.skipTests, 'true') }}:
+    - script: ${{ parameters.buildScript }}
+              -integrationTest
+              -configuration ${{ parameters.buildConfig }}
+              -warnAsError 1
+              /bl:${{ parameters.repoLogPath }}/integration_tests.binlog
+              $(_OfficialBuildIdArgs)
+      displayName: Run integration tests
 
     - pwsh: |
         $SourcesDirectory = '$(Build.SourcesDirectory)';
@@ -151,12 +169,11 @@ steps:
       displayName: Build Azure DevOps plugin
 
     - script: ${{ parameters.buildScript }}
-              -pack
               -sign $(_SignArgs)
               -publish $(_PublishArgs)
               -configuration ${{ parameters.buildConfig }}
               -warnAsError 1
-              /bl:${{ parameters.repoLogPath }}/pack.binlog
+              /bl:${{ parameters.repoLogPath }}/publish.binlog
               /p:Restore=false /p:Build=false
               $(_OfficialBuildIdArgs)
-      displayName: Pack, sign, and publish
+      displayName: Sign and publish

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs

@@ -0,0 +1,161 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.Extensions.AI.Templates.Tests;
+
+/// <summary>
+/// Contains execution tests for the "AI Chat Web" template.
+/// </summary>
+/// <remarks>
+/// In addition to validating that the templates build and restore correctly,
+/// these tests are also responsible for template component governance reporting.
+/// This is because the generated output is left on disk after tests complete,
+/// most importantly the project.assets.json file that gets created during restore.
+/// Therefore, it's *critical* that these tests remain in a working state,
+/// as disabling them will also disable CG reporting.
+/// </remarks>
+public class AIChatWebExecutionTests : TemplateExecutionTestBase<AIChatWebExecutionTests>, ITemplateExecutionTestConfigurationProvider
+{
+    public AIChatWebExecutionTests(TemplateExecutionTestFixture fixture, ITestOutputHelper outputHelper)
+        : base(fixture, outputHelper)
+    {
+    }
+
+    public static TemplateExecutionTestConfiguration Configuration { get; } = new()
+    {
+        TemplatePackageName = "Microsoft.Extensions.AI.Templates",
+        TestOutputFolderPrefix = "AIChatWeb"
+    };
+
+    public static IEnumerable<object[]> GetBasicTemplateOptions()
+        => GetFilteredTemplateOptions("--aspire", "false");
+
+    public static IEnumerable<object[]> GetAspireTemplateOptions()
+        => GetFilteredTemplateOptions("--aspire", "true");
+
+    // Do not skip. See XML docs for this test class.
+    [Theory]
+    [MemberData(nameof(GetBasicTemplateOptions))]
+    public async Task CreateRestoreAndBuild_BasicTemplate(params string[] args)
+    {
+        const string ProjectName = "BasicApp";
+        var project = await Fixture.CreateProjectAsync(
+            templateName: "aichatweb",
+            projectName: ProjectName,
+            args);
+
+        await Fixture.RestoreProjectAsync(project);
+        await Fixture.BuildProjectAsync(project);
+    }
+
+    // Do not skip. See XML docs for this test class.
+    [Theory]
+    [MemberData(nameof(GetAspireTemplateOptions))]
+    public async Task CreateRestoreAndBuild_AspireTemplate(params string[] args)
+    {
+        const string ProjectName = "AspireApp";
+        var project = await Fixture.CreateProjectAsync(
+            templateName: "aichatweb",
+            ProjectName,
+            args);
+
+        project.StartupProjectRelativePath = $"{ProjectName}.AppHost";
+
+        await Fixture.RestoreProjectAsync(project);
+        await Fixture.BuildProjectAsync(project);
+    }
+
+    private static readonly (string name, string[] values)[] _templateOptions = [
+        ("--provider",          ["azureopenai", "githubmodels", "ollama", "openai"]),
+        ("--vector-store",      ["azureaisearch", "local", "qdrant"]),
+        ("--managed-identity",  ["true", "false"]),
+        ("--aspire",            ["true", "false"]),
+    ];
+
+    private static IEnumerable<object[]> GetFilteredTemplateOptions(params string[] filter)
+    {
+        foreach (var options in GetAllPossibleOptions(_templateOptions))
+        {
+            if (!MatchesFilter())
+            {
+                continue;
+            }
+
+            if (HasOption("--managed-identity", "true"))
+            {
+                if (HasOption("--aspire", "true"))
+                {
+                    // The managed identity option is disabled for the Aspire template.
+                    continue;
+                }
+
+                if (!HasOption("--vector-store", "azureaisearch") &&
+                    !HasOption("--aspire", "false"))
+                {
+                    // Can only use managed identity when using Azure in the non-Aspire template.
+                    continue;
+                }
+            }
+
+            if (HasOption("--vector-store", "qdrant") &&
+                HasOption("--aspire", "false"))
+            {
+                // Can't use Qdrant without Aspire.
+                continue;
+            }
+
+            yield return options;
+
+            bool MatchesFilter()
+            {
+                for (var i = 0; i < filter.Length; i += 2)
+                {
+                    if (!HasOption(filter[i], filter[i + 1]))
+                    {
+                        return false;
+                    }
+                }
+
+                return true;
+            }
+
+            bool HasOption(string name, string value)
+            {
+                for (var i = 0; i < options.Length; i += 2)
+                {
+                    if (string.Equals(name, options[i], StringComparison.Ordinal) &&
+                        string.Equals(value, options[i + 1], StringComparison.Ordinal))
+                    {
+                        return true;
+                    }
+                }
+
+                return false;
+            }
+        }
+    }
+
+    private static IEnumerable<string[]> GetAllPossibleOptions(ReadOnlyMemory<(string name, string[] values)> options)
+    {
+        if (options.Length == 0)
+        {
+            yield return [];
+            yield break;
+        }
+
+        var first = options.Span[0];
+        foreach (var restSelection in GetAllPossibleOptions(options[1..]))
+        {
+            foreach (var value in first.values)
+            {
+                yield return [first.name, value, .. restSelection];
+            }
+        }
+    }
+}

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AichatwebTemplatesTests.cs → test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebSnapshotTests.cs

@@ -6,17 +6,16 @@
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
-using Microsoft.Extensions.AI.Templates.IntegrationTests;
 using Microsoft.Extensions.AI.Templates.Tests;
 using Microsoft.Extensions.Logging;
 using Microsoft.TemplateEngine.Authoring.TemplateVerifier;
 using Microsoft.TemplateEngine.TestHelper;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Microsoft.Extensions.AI.Templates.InegrationTests;
+namespace Microsoft.Extensions.AI.Templates.Tests;
 
-public class AichatwebTemplatesTests : TestBase
+public class AIChatWebSnapshotTests
 {
     // Keep the exclude patterns below in sync with those in Microsoft.Extensions.AI.Templates.csproj.
     private static readonly string[] _verificationExcludePatterns = [
@@ -36,7 +35,7 @@ public class AichatwebTemplatesTests : TestBase
 
     private readonly ILogger _log;
 
-    public AichatwebTemplatesTests(ITestOutputHelper log)
+    public AIChatWebSnapshotTests(ITestOutputHelper log)
     {
 #pragma warning disable CA2000 // Dispose objects before losing scope
         _log = new XunitLoggerProvider(log).CreateLogger("TestRun");
@@ -67,7 +66,7 @@ private async Task TestTemplateCoreAsync(string scenarioName, IEnumerable<string
         string templateShortName = "aichatweb";
 
         // Get the template location
-        string templateLocation = Path.Combine(TemplateFeedLocation, "Microsoft.Extensions.AI.Templates", "src", "ChatWithCustomData");
+        string templateLocation = Path.Combine(WellKnownPaths.TemplateFeedLocation, "Microsoft.Extensions.AI.Templates", "src", "ChatWithCustomData");
 
         var verificationExcludePatterns = Path.DirectorySeparatorChar is '/'
             ? _verificationExcludePatterns

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetCommand.cs

@@ -0,0 +1,19 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+namespace Microsoft.Extensions.AI.Templates.Tests;
+
+public class DotNetCommand : TestCommand
+{
+    public DotNetCommand(params ReadOnlySpan<string> args)
+    {
+        FileName = WellKnownPaths.RepoDotNetExePath;
+
+        foreach (var arg in args)
+        {
+            Arguments.Add(arg);
+        }
+    }
+}

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetNewCommand.cs

@@ -0,0 +1,39 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+
+namespace Microsoft.Extensions.AI.Templates.Tests;
+
+public sealed class DotNetNewCommand : DotNetCommand
+{
+    private bool _customHiveSpecified;
+
+    public DotNetNewCommand(params ReadOnlySpan<string> args)
+        : base(["new", .. args])
+    {
+    }
+
+    public DotNetNewCommand WithCustomHive(string path)
+    {
+        Arguments.Add("--debug:custom-hive");
+        Arguments.Add(path);
+        _customHiveSpecified = true;
+        return this;
+    }
+
+    public override Task<TestCommandResult> ExecuteAsync(ITestOutputHelper outputHelper)
+    {
+        if (!_customHiveSpecified)
+        {
+            // If this exception starts getting thrown in cases where a custom hive is
+            // legitimately undesirable, we can add a new 'WithoutCustomHive()' method that
+            // just sets '_customHiveSpecified' to 'true'.
+            throw new InvalidOperationException($"A {nameof(DotNetNewCommand)} should specify a custom hive with '{nameof(WithCustomHive)}()'.");
+        }
+
+        return base.ExecuteAsync(outputHelper);
+    }
+}

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateExecutionTestConfigurationProvider.cs

@@ -0,0 +1,9 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.Extensions.AI.Templates.Tests;
+
+public interface ITemplateExecutionTestConfigurationProvider
+{
+    static abstract TemplateExecutionTestConfiguration Configuration { get; }
+}

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/MessageSinkTestOutputHelper.cs

@@ -0,0 +1,27 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Microsoft.Extensions.AI.Templates.Tests;
+
+public sealed class MessageSinkTestOutputHelper : ITestOutputHelper
+{
+    private readonly IMessageSink _messageSink;
+
+    public MessageSinkTestOutputHelper(IMessageSink messageSink)
+    {
+        _messageSink = messageSink;
+    }
+
+    public void WriteLine(string message)
+    {
+        _messageSink.OnMessage(new DiagnosticMessage(message));
+    }
+
+    public void WriteLine(string format, params object[] args)
+    {
+        _messageSink.OnMessage(new DiagnosticMessage(format, args));
+    }
+}

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ProcessExtensions.cs

@@ -0,0 +1,19 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Diagnostics;
+
+public static class ProcessExtensions
+{
+    public static bool TryGetHasExited(this Process process)
+    {
+        try
+        {
+            return process.HasExited;
+        }
+        catch (InvalidOperationException ex) when (ex.Message.Contains("No process is associated with this object"))
+        {
+            return true;
+        }
+    }
+}