address code review feedback

Author: adamsitnik

src/System.CommandLine.Tests/DirectiveTests.cs

@@ -11,15 +11,13 @@ namespace System.CommandLine.Tests
     public class DirectiveTests
     {
         [Fact]
-        public void Directives_should_not_be_considered_as_unmatched_tokens_when_they_are_enabled()
+        public void Directives_should_be_considered_as_unmatched_tokens_when_they_are_not_matched()
         {
-            RootCommand root = new () { new Option<bool>("-y") };
-            CommandLineBuilder builder = new (root);
-            builder.Directives.Add(new ("some"));
+            Directive directive = new("parse");
 
-            var result = root.Parse($"{RootCommand.ExecutableName} [parse] -y", builder.Build());
+            ParseResult result = Parse(new Option<bool>("-y"), directive, $"{RootCommand.ExecutableName} [nonExisting] -y");
 
-            result.UnmatchedTokens.Should().BeEmpty();
+            result.UnmatchedTokens.Should().ContainSingle("[nonExisting]");
         }
 
         [Fact]
@@ -99,14 +97,19 @@ public void Directives_must_have_a_non_empty_key(string directive)
         }
 
         [Theory]
-        [InlineData("[par se]")]
-        [InlineData("[ parse]")]
-        [InlineData("[parse ]")]
-        public void Directives_cannot_contain_spaces(string value)
+        [InlineData("[par se]", "[par", "se]")]
+        [InlineData("[ parse]", "[", "parse]")]
+        [InlineData("[parse ]", "[parse", "]")]
+        public void Directives_cannot_contain_spaces(string value, string firstUnmatchedToken, string secondUnmatchedToken)
         {
             Action create = () => new Directive(value);
-
             create.Should().Throw<ArgumentException>();
+
+            Directive directive = new("parse");
+            ParseResult result = Parse(new Option<bool>("-y"), directive, $"{value} -y");
+            result.FindResultFor(directive).Should().BeNull();
+
+            result.UnmatchedTokens.Should().BeEquivalentTo(firstUnmatchedToken, secondUnmatchedToken);
         }
 
         [Fact]
@@ -119,24 +122,6 @@ public void When_a_directive_is_specified_more_than_once_then_its_values_are_agg
             result.FindResultFor(directive).Values.Should().BeEquivalentTo("one", "two");
         }
 
-        [Fact]
-        public void When_directives_are_not_enabled_they_are_treated_as_regular_tokens()
-        {
-            var config = new CommandLineConfiguration(
-                    new RootCommand
-                    {
-                        new Argument<List<string>>()
-                    });
-
-            var result = config.RootCommand.Parse("[hello]", config);
-
-            result.CommandResult
-                  .Tokens
-                  .Select(t => t.Value)
-                  .Should()
-                  .BeEquivalentTo("[hello]");
-        }
-
         private static ParseResult Parse(Option option, Directive directive, string commandLine)
         {
             RootCommand root = new() { option };

src/System.CommandLine/Parsing/ParseOperation.cs

@@ -297,7 +297,7 @@ void ParseDirective()
 
                 if (token.Symbol is not Directive directive)
                 {
-                    // Directives_should_not_be_considered_as_unmatched_tokens
+                    AddCurrentTokenToUnmatched();
                     return;
                 }
 

src/System.CommandLine/Parsing/StringExtensions.cs

@@ -82,11 +82,11 @@ internal static void Tokenize(
 
             var currentCommand = configuration.RootCommand;
             var foundDoubleDash = false;
-            var foundEndOfDirectives = configuration.Directives.Count == 0;
+            var foundEndOfDirectives = false;
 
             var tokenList = new List<Token>(args.Count);
 
-            var knownTokens = configuration.RootCommand.ValidTokens(configuration);
+            var knownTokens = configuration.RootCommand.ValidTokens(configuration.Directives);
 
             int i = FirstArgumentIsRootCommand(args, configuration.RootCommand, inferRootCommand)
                 ? 0
@@ -185,7 +185,7 @@ configuration.TokenReplacer is { } replacer &&
                                     if (cmd != configuration.RootCommand)
                                     {
                                         knownTokens = cmd.ValidTokens(
-                                            configuration: null); // config contains Directives, they are allowed only for RootCommand
+                                            directives: null); // config contains Directives, they are allowed only for RootCommand
                                     }
                                     currentCommand = cmd;
                                     tokenList.Add(Command(arg, cmd));
@@ -431,15 +431,15 @@ static IEnumerable<string> SplitLine(string line)
             }
         }
 
-        private static Dictionary<string, Token> ValidTokens(this Command command, CommandLineConfiguration? configuration)
+        private static Dictionary<string, Token> ValidTokens(this Command command, IReadOnlyList<Directive>? directives)
         {
             Dictionary<string, Token> tokens = new(StringComparer.Ordinal);
 
-            if (configuration?.Directives is not null)
+            if (directives is not null)
             {
-                for (int directiveIndex = 0; directiveIndex < configuration.Directives.Count; directiveIndex++)
+                for (int directiveIndex = 0; directiveIndex < directives.Count; directiveIndex++)
                 {
-                    Directive directive = configuration.Directives[directiveIndex];
+                    Directive directive = directives[directiveIndex];
                     tokens.Add(
                         directive.Name,
                         new Token(directive.Name, TokenType.Directive, directive, Token.ImplicitPosition));