-
Notifications
You must be signed in to change notification settings - Fork 378
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Initial implementation of the test command and API #5263
Merged
JanKrivanek
merged 15 commits into
dotnet:feature/test-framework
from
JanKrivanek:test-framework-inception
Oct 10, 2022
Merged
Changes from 14 commits
Commits
Show all changes
15 commits
Select commit
Hold shift + click to select a range
abdf04b
Initial implementation of the test command and API
JanKrivanek 8d398fe
Disable diff tool in CI
JanKrivanek 48320b7
Use new Verifier, Add verification options
JanKrivanek 1fde346
Add 3rd party signing info
JanKrivanek fe2f6b2
Remove dependency on test project
JanKrivanek b993d78
Use custom exceptions for exit expectations failed
JanKrivanek d8c79d1
Improve engine API, add API tests
JanKrivanek ba252cd
Add localizations, move types
JanKrivanek f90e35d
Various improvements for PR
JanKrivanek 6e5f69a
Add the uninstalled template testing option
JanKrivanek bced6df
Add integration tests and unit tests
JanKrivanek 25018fc
Add VerifyEngine unit tests, add ability to place expectations in cod…
JanKrivanek e0d49da
Add sample usage in unit test
JanKrivanek 0513f64
Adjust assertions (OS agnostic)
JanKrivanek 9c4c7db
Fix code locations
JanKrivanek File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
204 changes: 204 additions & 0 deletions
204
src/Microsoft.TemplateEngine.Authoring.CLI/Commands/Verify/VerifyCommand.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,204 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using System.CommandLine; | ||
| using System.CommandLine.Binding; | ||
| using System.CommandLine.Parsing; | ||
| using Microsoft.Extensions.Logging; | ||
| using Microsoft.Extensions.Logging.Abstractions; | ||
| using Microsoft.TemplateEngine.Authoring.TemplateVerifier; | ||
|
|
||
| namespace Microsoft.TemplateEngine.Authoring.CLI.Commands.Verify | ||
| { | ||
| internal class VerifyCommand : ExecutableCommand<VerifyCommandArgs> | ||
| { | ||
| private const string CommandName = "verify"; | ||
|
|
||
| private readonly Argument<string> _templateNameArgument = new("template-short-name") | ||
| { | ||
| Description = LocalizableStrings.command_verify_help_templateName_description, | ||
| // 0 for case where only path is specified | ||
| Arity = new ArgumentArity(1, 1) | ||
| }; | ||
|
|
||
| private readonly Option<string> _remainingArguments = new Option<string>("--template-args") | ||
| { | ||
| Description = "Template specific arguments - all joined into single enquoted string. Any needed quotations of actual arguments has to be escaped.", | ||
| Arity = new ArgumentArity(0, 1) | ||
| }; | ||
|
|
||
| private readonly Option<string> _templatePathOption = new(new[] { "-p", "--template-path" }) | ||
| { | ||
| Description = LocalizableStrings.command_verify_help_templatePath_description, | ||
| }; | ||
|
|
||
| private readonly Option<string> _newCommandPathOption = new("--new-command-assembly") | ||
| { | ||
| Description = LocalizableStrings.command_verify_help_newCommandPath_description, | ||
| //TODO: do we have better way of distinguishing options that might rarely be needed? | ||
| // if not - we should probably add a link to more detailed help in the command description (mentioning that online help has additional options) | ||
| IsHidden = true | ||
| }; | ||
|
|
||
| private readonly Option<string> _templateOutputPathOption = new(new[] { "-o", "--output" }) | ||
| { | ||
| Description = LocalizableStrings.command_verify_help_outputPath_description, | ||
| }; | ||
|
|
||
| private readonly Option<string> _expectationsDirectoryOption = new(new[] { "-d", "--expectations-directory" }) | ||
| { | ||
| Description = LocalizableStrings.command_verify_help_expectationsDirPath_description, | ||
| }; | ||
|
|
||
| private readonly Option<bool> _disableDiffToolOption = new("--disable-diff-tool") | ||
| { | ||
| Description = LocalizableStrings.command_verify_help_disableDiffTool_description, | ||
| }; | ||
|
|
||
| private readonly Option<bool> _disableDefaultExcludePatternsOption = new("--disable-default-exclude-patterns") | ||
| { | ||
| Description = LocalizableStrings.command_verify_help_disableDefaultExcludes_description, | ||
| }; | ||
|
|
||
| private readonly Option<IEnumerable<string>> _excludePatternOption = new("--exclude-pattern") | ||
| { | ||
| Description = LocalizableStrings.command_verify_help_customExcludes_description, | ||
| Arity = new ArgumentArity(0, 999) | ||
| }; | ||
|
|
||
| private readonly Option<bool> _verifyCommandOutputOption = new("--verify-std") | ||
| { | ||
| Description = LocalizableStrings.command_verify_help_verifyOutputs_description, | ||
| }; | ||
|
|
||
| private readonly Option<bool> _isCommandExpectedToFailOption = new("--fail-expected") | ||
| { | ||
| Description = LocalizableStrings.command_verify_help_expectFailure_description, | ||
| }; | ||
|
|
||
| private readonly Option<IEnumerable<UniqueForOption>> _uniqueForOption = new("--unique-for") | ||
| { | ||
| Description = LocalizableStrings.command_verify_help_uniqueFor_description, | ||
| Arity = new ArgumentArity(0, 999), | ||
| AllowMultipleArgumentsPerToken = true, | ||
| }; | ||
|
|
||
| public VerifyCommand(ILoggerFactory loggerFactory) | ||
| : base(CommandName, LocalizableStrings.command_verify_help_description, loggerFactory) | ||
| { | ||
| AddArgument(_templateNameArgument); | ||
| AddOption(_remainingArguments); | ||
| AddOption(_templatePathOption); | ||
| AddOption(_newCommandPathOption); | ||
| AddOption(_templateOutputPathOption); | ||
| AddOption(_expectationsDirectoryOption); | ||
| AddOption(_disableDiffToolOption); | ||
| AddOption(_disableDefaultExcludePatternsOption); | ||
| AddOption(_excludePatternOption); | ||
| AddOption(_verifyCommandOutputOption); | ||
| AddOption(_isCommandExpectedToFailOption); | ||
| FromAmongCaseInsensitive( | ||
| _uniqueForOption, | ||
| System.Enum.GetNames(typeof(UniqueForOption)) | ||
| .Where(v => !v.Equals(UniqueForOption.None.ToString(), StringComparison.OrdinalIgnoreCase)) | ||
| .ToArray()); | ||
JanKrivanek marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| AddOption(_uniqueForOption); | ||
| } | ||
|
|
||
| internal static VerifyCommandArgs ExtractArguments(VerifyCommand verifyCommand, ParseResult parseResult) | ||
| { | ||
| return new VerifyCommandArgs( | ||
| templateName: parseResult.GetValueForArgument(verifyCommand._templateNameArgument), | ||
| templateSpecificArgs: parseResult.GetValueForOption(verifyCommand._remainingArguments), | ||
| templatePath: parseResult.GetValueForOption(verifyCommand._templatePathOption), | ||
| dotnetNewCommandAssemblyPath: parseResult.GetValueForOption(verifyCommand._newCommandPathOption), | ||
| expectationsDirectory: parseResult.GetValueForOption(verifyCommand._expectationsDirectoryOption), | ||
| outputDirectory: parseResult.GetValueForOption(verifyCommand._templateOutputPathOption), | ||
| disableDiffTool: parseResult.GetValueForOption(verifyCommand._disableDiffToolOption), | ||
| disableDefaultVerificationExcludePatterns: parseResult.GetValueForOption(verifyCommand._disableDefaultExcludePatternsOption), | ||
| verificationExcludePatterns: parseResult.GetValueForOption(verifyCommand._excludePatternOption), | ||
| verifyCommandOutput: parseResult.GetValueForOption(verifyCommand._verifyCommandOutputOption), | ||
| isCommandExpectedToFail: parseResult.GetValueForOption(verifyCommand._isCommandExpectedToFailOption), | ||
| uniqueForOptions: parseResult.GetValueForOption(verifyCommand._uniqueForOption)); | ||
| } | ||
|
|
||
| protected override async Task<int> ExecuteAsync(VerifyCommandArgs args, CancellationToken cancellationToken = default) | ||
| { | ||
| Logger.LogInformation("Running the verification of {templateName}.", args.TemplateName); | ||
|
|
||
| try | ||
| { | ||
| VerificationEngine engine = new VerificationEngine(LoggerFactory ?? NullLoggerFactory.Instance); | ||
| TemplateVerifierOptions options = new(templateName: args.TemplateName) | ||
| { | ||
| TemplatePath = args.TemplatePath, | ||
| TemplateSpecificArgs = args.TemplateSpecificArgs, | ||
| DisableDiffTool = args.DisableDiffTool, | ||
| DisableDefaultVerificationExcludePatterns = args.DisableDefaultVerificationExcludePatterns, | ||
| VerificationExcludePatterns = args.VerificationExcludePatterns, | ||
| DotnetNewCommandAssemblyPath = args.DotnetNewCommandAssemblyPath, | ||
| ExpectationsDirectory = args.ExpectationsDirectory, | ||
| OutputDirectory = args.OutputDirectory, | ||
| VerifyCommandOutput = args.VerifyCommandOutput, | ||
| IsCommandExpectedToFail = args.IsCommandExpectedToFail, | ||
| UniqueFor = args.UniqueFor, | ||
| }; | ||
| await engine.Execute( | ||
| options, | ||
| cancellationToken, | ||
| // We explicitly pass a path - so that the engine then process it and gets the current executing dir | ||
| // and treats it as a code base of caller of API (as in case of CLI usage we do not want to store | ||
| // expectation files in CLI sources dir) | ||
| Path.Combine(Environment.CurrentDirectory, "_") | ||
| ).ConfigureAwait(false); | ||
| return 0; | ||
| } | ||
| catch (Exception e) | ||
| { | ||
| Logger.LogError(LocalizableStrings.command_verify_error_failed); | ||
| Logger.LogError(e.Message); | ||
| TemplateVerificationException? ex = e as TemplateVerificationException; | ||
| return (int)(ex?.TemplateVerificationErrorCode ?? TemplateVerificationErrorCode.InternalError); | ||
| } | ||
| } | ||
|
|
||
| protected override BinderBase<VerifyCommandArgs> GetModelBinder() => new VerifyModelBinder(this); | ||
|
|
||
| /// <summary> | ||
| /// Case insensitive version for <see cref="System.CommandLine.OptionExtensions.FromAmong{TOption}(TOption, string[])"/>. | ||
| /// </summary> | ||
| private static void FromAmongCaseInsensitive(Option option, string[]? allowedValues = null, string? allowedHiddenValue = null) | ||
| { | ||
| allowedValues ??= Array.Empty<string>(); | ||
| option.AddValidator(optionResult => ValidateAllowedValues(optionResult, allowedValues, allowedHiddenValue)); | ||
| option.AddCompletions(allowedValues); | ||
| } | ||
|
|
||
| private static void ValidateAllowedValues(OptionResult optionResult, string[] allowedValues, string? allowedHiddenValue = null) | ||
| { | ||
| var invalidArguments = optionResult.Tokens.Where(token => !allowedValues.Append(allowedHiddenValue).Contains(token.Value, StringComparer.OrdinalIgnoreCase)).ToList(); | ||
| if (invalidArguments.Any()) | ||
| { | ||
| optionResult.ErrorMessage = string.Format( | ||
| LocalizableStrings.command_verify_error_unrecognizedArguments, | ||
| string.Join(", ", invalidArguments.Select(arg => $"'{arg.Value}'")), | ||
| string.Join(", ", allowedValues.Select(allowedValue => $"'{allowedValue}'"))); | ||
| } | ||
| } | ||
|
|
||
| private class VerifyModelBinder : BinderBase<VerifyCommandArgs> | ||
| { | ||
| private readonly VerifyCommand _verifyCommand; | ||
|
|
||
| internal VerifyModelBinder(VerifyCommand verifyCommand) | ||
| { | ||
| _verifyCommand = verifyCommand; | ||
| } | ||
|
|
||
| protected override VerifyCommandArgs GetBoundValue(BindingContext bindingContext) | ||
| { | ||
| return VerifyCommand.ExtractArguments(_verifyCommand, bindingContext.ParseResult); | ||
| } | ||
| } | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
just curious: why this was not needed before?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@vlada-shubina https://github.com/VerifyTests/Verify/blob/main/docs/upgrade17-18.md#move-to-argon