logging validation messages

src/Microsoft.TemplateEngine.Edge/Settings/Scanner.cs

@@ -6,6 +6,7 @@
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 #if !NETFULL
@@ -214,16 +215,15 @@ private async Task<ScanResult> ScanMountPointForTemplatesAsync(MountPointScanSou
             foreach (IGenerator generator in _environmentSettings.Components.OfType<IGenerator>())
             {
                 IReadOnlyList<IScanTemplateInfo> templateList = await generator.GetTemplatesFromMountPointAsync(source.MountPoint, cancellationToken).ConfigureAwait(false);
-                foreach (IScanTemplateInfo template in templateList)
-                {
-                    templates.Add(template);
-                }
+                LogScanningResults(source, templateList, generator);
 
-                source.FoundTemplates |= templateList.Count > 0;
+                IEnumerable<IScanTemplateInfo> validTemplates = templateList.Where(t => t.IsValid);
+                templates.AddRange(validTemplates);
+                source.FoundTemplates |= validTemplates.Any();
             }
 
             //backward compatibility
-            var localizationLocators = templates.SelectMany(t => t.Localizations.Values).ToList();
+            var localizationLocators = templates.SelectMany(t => t.Localizations.Values.Where(li => li.IsValid)).ToList();
             return new ScanResult(source.MountPoint, templates, localizationLocators, Array.Empty<(string, Type, IIdentifiedComponent)>());
         }
 
@@ -281,6 +281,96 @@ private IEnumerable<KeyValuePair<string, Assembly>> LoadAllFromPath(
             return loaded;
         }
 
+        private void LogScanningResults(MountPointScanSource source, IReadOnlyList<IScanTemplateInfo> foundTemplates, IGenerator generator)
+        {
+            ILogger logger = _environmentSettings.Host.Logger;
+            logger.LogDebug("Scanning mount point '{0}' by generator '{1}': found {2} templates", source.MountPoint.MountPointUri, generator.Id, foundTemplates.Count);
+            foreach (IScanTemplateInfo template in foundTemplates)
+            {
+                string templateDisplayName = GetTemplateDisplayName(template);
+                logger.LogDebug("Found template {0}", templateDisplayName);
+
+                LogValidationEntries(
+                    logger,
+                    string.Format("The template {0} has the following validation errors:", templateDisplayName),
+                    template.ValidationErrors,
+                    IValidationEntry.SeverityLevel.Error);
+                LogValidationEntries(
+                    logger,
+                    string.Format("The template {0} has the following validation warnings:", templateDisplayName),
+                    template.ValidationErrors,
+                    IValidationEntry.SeverityLevel.Warning);
+                LogValidationEntries(
+                    logger,
+                    string.Format("The template {0} has the following validation messages:", templateDisplayName),
+                    template.ValidationErrors,
+                    IValidationEntry.SeverityLevel.Info);
+
+                foreach (KeyValuePair<string, ILocalizationLocator> locator in template.Localizations)
+                {
+                    ILocalizationLocator localizationInfo = locator.Value;
+
+                    LogValidationEntries(
+                        logger,
+                        string.Format("The template {0} has the following validation errors in '{1}' localization:", templateDisplayName, localizationInfo.Locale),
+                        localizationInfo.ValidationErrors,
+                        IValidationEntry.SeverityLevel.Error);
+                    LogValidationEntries(
+                        logger,
+                        string.Format("The template {0} has the following validation warnings in '{1}' localization:", templateDisplayName, localizationInfo.Locale),
+                        localizationInfo.ValidationErrors,
+                        IValidationEntry.SeverityLevel.Warning);
+                    LogValidationEntries(
+                        logger,
+                        string.Format("The template {0} has the following validation messages in '{1}' localization:", templateDisplayName, localizationInfo.Locale),
+                        localizationInfo.ValidationErrors,
+                        IValidationEntry.SeverityLevel.Info);
+                }
+
+                if (!template.IsValid)
+                {
+                    logger.LogError("Failed to install the template {0}: the template is not valid.", templateDisplayName);
+                }
+                foreach (ILocalizationLocator invalidLoc in template.Localizations.Values.Where(li => !li.IsValid))
+                {
+                    logger.LogWarning("Failed to install the '{0}' localization the template {1}: the localization file is not valid. The localization will be skipped.", invalidLoc.Locale, templateDisplayName);
+                }
+            }
+
+            static string GetTemplateDisplayName(IScanTemplateInfo template)
+            {
+                string templateName = string.IsNullOrEmpty(template.Name) ? "<no name>" : template.Name;
+                return $"'{templateName}' ({template.Identity})";
+            }
+
+            static string PrintError(IValidationEntry error) => $"   [{error.Severity}][{error.Code}] {error.ErrorMessage}";
+
+            static void LogValidationEntries(ILogger logger, string header, IReadOnlyList<IValidationEntry> errors, IValidationEntry.SeverityLevel severity)
+            {
+                Action<string> log = severity switch
+                {
+                    IValidationEntry.SeverityLevel.None => (string s) => throw new NotSupportedException($"{IValidationEntry.SeverityLevel.None} severity is not supported."),
+                    IValidationEntry.SeverityLevel.Info => (string s) => logger.LogDebug(s),
+                    IValidationEntry.SeverityLevel.Warning => (string s) => logger.LogWarning(s),
+                    IValidationEntry.SeverityLevel.Error => (string s) => logger.LogError(s),
+                    _ => throw new InvalidOperationException($"{severity} is not expected value for {nameof(IValidationEntry.SeverityLevel)}."),
+                };
+
+                if (!errors.Any(e => e.Severity == severity))
+                {
+                    return;
+                }
+
+                StringBuilder sb = new();
+                sb.AppendLine(header);
+                foreach (IValidationEntry error in errors.Where(e => e.Severity == severity))
+                {
+                    sb.AppendLine(PrintError(error));
+                }
+                log(sb.ToString());
+            }
+        }
+
         private class MountPointScanSource
         {
             public MountPointScanSource(string location, IMountPoint mountPoint, bool shouldStayInOriginalLocation, bool foundComponents, bool foundTemplates)

src/Microsoft.TemplateEngine.Orchestrator.RunnableProjects/DirectoryBasedTemplate.Interfaces.cs

@@ -33,7 +33,7 @@ internal abstract partial class DirectoryBasedTemplate : ITemplateMetadata, ITem
 
         int ITemplateMetadata.Precedence => ConfigurationModel.Precedence;
 
-        string ITemplateMetadata.Name => ConfigurationModel.Name ?? throw new TemplateValidationException("Template configuration should have name defined");
+        string ITemplateMetadata.Name => ConfigurationModel.Name ?? string.Empty;
 
         IReadOnlyList<string> ITemplateMetadata.ShortNameList => ConfigurationModel.ShortNameList ?? new List<string>();
 

src/Microsoft.TemplateEngine.Orchestrator.RunnableProjects/DirectoryBasedTemplate.cs

@@ -36,7 +36,7 @@ internal abstract partial class DirectoryBasedTemplate
         /// <summary>
         /// Creates the instance of the class based on configuration from <paramref name="templateFile"/>.
         /// </summary>
-        /// <exception cref="TemplateValidationException">when template configuration is invalid.</exception>
+        /// <exception cref="TemplateAuthoringException">when template configuration is invalid.</exception>
         /// <exception cref="InvalidOperationException">when template identity is null.</exception>
         /// <exception cref="NotSupportedException">when the template is not supported by current generator version.</exception>
         protected DirectoryBasedTemplate(IEngineEnvironmentSettings settings, IGenerator generator, IFile templateFile, string? baselineName = null)
@@ -50,7 +50,7 @@ protected DirectoryBasedTemplate(IEngineEnvironmentSettings settings, IGenerator
 
             if (ConfigFile.Parent?.Parent is null)
             {
-                throw new TemplateValidationException(LocalizableStrings.Authoring_TemplateRootOutsideInstallSource);
+                throw new TemplateAuthoringException(LocalizableStrings.Authoring_TemplateRootOutsideInstallSource);
             }
             ConfigDirectory = templateFile.Parent;
             TemplateSourceRoot = ConfigFile.Parent.Parent;

src/Microsoft.TemplateEngine.Orchestrator.RunnableProjects/RunnableProjectGenerator.cs

@@ -78,11 +78,6 @@ async Task<IReadOnlyList<IScanTemplateInfo>> IGenerator.GetTemplatesFromMountPoi
             {
                 throw;
             }
-            catch (TemplateValidationException)
-            {
-                //do nothing
-                //template validation prints all required information
-            }
             catch (NotSupportedException ex)
             {
                 //do not print stack trace for this type.
@@ -280,11 +275,6 @@ bool IGenerator.TryGetTemplateFromConfigInfo(IFileSystemInfo templateFileConfig,
                 template = loadedTemplate;
                 return true;
             }
-            catch (TemplateValidationException)
-            {
-                //do nothing
-                //template validation prints all required information
-            }
             catch (NotSupportedException ex)
             {
                 //do not print stack trace for this type.
@@ -360,11 +350,6 @@ internal async Task<IReadOnlyList<ScannedTemplateInfo>> GetTemplatesFromMountPoi
                     cancellationToken.ThrowIfCancellationRequested();
                     templateList.Add(discoveredTemplate);
                 }
-                catch (TemplateValidationException)
-                {
-                    //do nothing
-                    //template validation prints all required information
-                }
                 catch (NotSupportedException ex)
                 {
                     //do not print stack trace for this type.

test/Microsoft.TemplateEngine.Edge.UnitTests/ScannerTests.cs

@@ -0,0 +1,145 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.Extensions.Logging;
+using Microsoft.TemplateEngine.Abstractions;
+using Microsoft.TemplateEngine.Edge.Settings;
+using Microsoft.TemplateEngine.TestHelper;
+using Microsoft.TemplateEngine.Tests;
+using Xunit;
+
+namespace Microsoft.TemplateEngine.Edge.UnitTests
+{
+    public class ScannerTests : TestBase, IClassFixture<EnvironmentSettingsHelper>
+    {
+        private readonly EnvironmentSettingsHelper _settingsHelper;
+
+        public ScannerTests(EnvironmentSettingsHelper environmentSettingsHelper)
+        {
+            _settingsHelper = environmentSettingsHelper;
+        }
+
+        [Fact]
+        public async Task CanLogValidationMessagesOnInstall_MissingIdentity()
+        {
+            string templatesLocation = Path.Combine(TestTemplatesLocation, "Invalid", "MissingIdentity");
+
+            List<(LogLevel Level, string Message)> loggedMessages = new();
+            InMemoryLoggerProvider loggerProvider = new(loggedMessages);
+
+            IEngineEnvironmentSettings settings = _settingsHelper.CreateEnvironment(virtualize: true, addLoggerProviders: new[] { loggerProvider });
+
+            Scanner scanner = new(settings);
+
+            ScanResult result = await scanner.ScanAsync(templatesLocation, default).ConfigureAwait(false);
+
+            Assert.Equal(0, result.Templates.Count);
+#pragma warning disable CS0618 // Type or member is obsolete
+            Assert.Equal(0, result.Localizations.Count);
+#pragma warning restore CS0618 // Type or member is obsolete
+
+            string errorMessage = Assert.Single(loggedMessages, l => l.Level is LogLevel.Error).Message;
+            Assert.Equal($"Failed to load template from {Path.GetFullPath(templatesLocation) + Path.DirectorySeparatorChar}.template.config/template.json.{Environment.NewLine}Details: 'identity' is missing or is an empty string.", errorMessage);
+        }
+
+        [Fact]
+        public async Task CanLogValidationMessagesOnInstall_ErrorsInTemplateConfig()
+        {
+            string templatesLocation = Path.Combine(TestTemplatesLocation, "Invalid", "MissingMandatoryConfig");
+
+            List<(LogLevel Level, string Message)> loggedMessages = new();
+            InMemoryLoggerProvider loggerProvider = new(loggedMessages);
+
+            IEngineEnvironmentSettings settings = _settingsHelper.CreateEnvironment(virtualize: true, addLoggerProviders: new[] { loggerProvider });
+
+            Scanner scanner = new(settings);
+
+            ScanResult result = await scanner.ScanAsync(templatesLocation, default).ConfigureAwait(false);
+
+            Assert.Equal(0, result.Templates.Count);
+#pragma warning disable CS0618 // Type or member is obsolete
+            Assert.Equal(0, result.Localizations.Count);
+#pragma warning restore CS0618 // Type or member is obsolete
+
+            List<string> errorMessages = loggedMessages.Where(lm => lm.Level == LogLevel.Error).Select(e => e.Message).ToList();
+            Assert.Equal(2, errorMessages.Count);
+
+            List<string> warningMessages = loggedMessages.Where(lm => lm.Level == LogLevel.Warning).Select(e => e.Message).ToList();
+            Assert.Empty(warningMessages);
+
+            List<string> debugMessages = loggedMessages.Where(lm => lm.Level == LogLevel.Debug).Select(e => e.Message).ToList();
+            Assert.Equal(4, debugMessages.Count);
+
+            Assert.Equal(
+                """
+                The template '<no name>' (MissingConfigTest) has the following validation errors:
+                   [Error][MV002] Missing 'name'.
+                   [Error][MV003] Missing 'shortName'.
+
+                """,
+                errorMessages[0]);
+            Assert.Equal("Failed to install the template '<no name>' (MissingConfigTest): the template is not valid.", errorMessages[1]);
+
+            Assert.Contains(
+                """
+                The template '<no name>' (MissingConfigTest) has the following validation messages:
+                   [Info][MV005] Missing 'sourceName'.
+                   [Info][MV006] Missing 'author'.
+                   [Info][MV007] Missing 'groupIdentity'.
+                   [Info][MV008] Missing 'generatorVersions'.
+                   [Info][MV009] Missing 'precedence'.
+                   [Info][MV010] Missing 'classifications'.
+
+                """,
+                debugMessages);
+        }
+
+        [Fact]
+        public async Task CanLogValidationMessagesOnInstall_Localization()
+        {
+            string templatesLocation = Path.Combine(TestTemplatesLocation, "Invalid", "Localization", "ValidationFailure");
+
+            List<(LogLevel Level, string Message)> loggedMessages = new();
+            InMemoryLoggerProvider loggerProvider = new(loggedMessages);
+
+            IEngineEnvironmentSettings settings = _settingsHelper.CreateEnvironment(virtualize: true, addLoggerProviders: new[] { loggerProvider });
+
+            Scanner scanner = new(settings);
+
+            ScanResult result = await scanner.ScanAsync(templatesLocation, default).ConfigureAwait(false);
+
+            Assert.Equal(1, result.Templates.Count);
+#pragma warning disable CS0618 // Type or member is obsolete
+            Assert.Equal(0, result.Localizations.Count);
+#pragma warning restore CS0618 // Type or member is obsolete
+
+            List<string> errorMessages = loggedMessages.Where(lm => lm.Level == LogLevel.Error).Select(e => e.Message).ToList();
+
+            Assert.Equal(2, errorMessages.Count);
+
+            List<string> warningMessages = loggedMessages.Where(lm => lm.Level == LogLevel.Warning).Select(e => e.Message).ToList();
+
+            Assert.Equal(3, warningMessages.Count);
+
+            Assert.Equal(
+               """
+                The template 'name' (TestAssets.Invalid.Localization.ValidationFailure) has the following validation errors in 'de-DE' localization:
+                   [Error][LOC001] In localization file under the post action with id 'pa1', there are localized strings for manual instruction(s) with ids 'do-not-exist'. These manual instructions do not exist in the template.json file and should be removed from localization file.
+                   [Error][LOC002] Post action(s) with id(s) 'pa0' specified in the localization file do not exist in the template.json file. Remove the localized strings from the localization file.
+
+                """,
+               errorMessages[0]);
+            Assert.Equal(
+                """
+                The template 'name' (TestAssets.Invalid.Localization.ValidationFailure) has the following validation errors in 'tr' localization:
+                   [Error][LOC002] Post action(s) with id(s) 'pa6' specified in the localization file do not exist in the template.json file. Remove the localized strings from the localization file.
+
+                """,
+                errorMessages[1]);
+
+            Assert.Equal($"[{Path.GetFullPath(templatesLocation) + Path.DirectorySeparatorChar}.template.config/template.json]: id of the post action 'pa2' at index '3' is not unique. Only the first post action that uses this id will be localized.", warningMessages[0]);
+            Assert.Equal("Failed to install the 'de-DE' localization the template 'name' (TestAssets.Invalid.Localization.ValidationFailure): the localization file is not valid. The localization will be skipped.", warningMessages[1]);
+            Assert.Equal("Failed to install the 'tr' localization the template 'name' (TestAssets.Invalid.Localization.ValidationFailure): the localization file is not valid. The localization will be skipped.", warningMessages[2]);
+        }
+    }
+}

test/Microsoft.TemplateEngine.Orchestrator.RunnableProjects.UnitTests/SchemaTests/JSONSchemaTests.cs

@@ -44,8 +44,12 @@ public void IsJSONSchemaValid(string testFile)
 
         public static IEnumerable<object?[]> GetAllTemplates()
         {
+            //those templates are intentionally wrong
+            string[] exceptions = new[] { "MissingIdentity", "MissingMandatoryConfig" };
+
             return Directory.EnumerateFiles(TestTemplatesLocation, "template.json", SearchOption.AllDirectories)
                 .Where(s => s.Contains(".template.config"))
+                .Where(s => !exceptions.Any(e => s.Contains(e)))
                 .Select(s => s.Remove(s.Length - JsonLocation.Length).Remove(0, TestTemplatesLocation.Length).Trim(Path.DirectorySeparatorChar))
                 .Select(s => new object?[] { s });
         }

test/Microsoft.TemplateEngine.Orchestrator.RunnableProjects.UnitTests/TemplateConfigTests/TemplateRootTests.cs

@@ -10,6 +10,7 @@
 using Microsoft.TemplateEngine.Abstractions.Mount;
 using Microsoft.TemplateEngine.Orchestrator.RunnableProjects.Validation;
 using Microsoft.TemplateEngine.TestHelper;
+using Microsoft.TemplateEngine.Utils;
 using Xunit;
 
 namespace Microsoft.TemplateEngine.Orchestrator.RunnableProjects.UnitTests.TemplateConfigTests
@@ -90,7 +91,7 @@ public void TemplateJsonCannotBeInMountPointRoot()
             IFile? templateConfigFile = mountPoint.FileInfo(pathToTemplateJson);
             Assert.NotNull(templateConfigFile);
 
-            TemplateValidationException e = Assert.Throws<TemplateValidationException>(() => new RunnableProjectConfig(environmentSettings, generator, templateConfigFile));
+            TemplateAuthoringException e = Assert.Throws<TemplateAuthoringException>(() => new RunnableProjectConfig(environmentSettings, generator, templateConfigFile));
             Assert.Equal(expectedErrorMessage, e.Message);
         }