Run the tokenizer as normal after the T_FN backfill

The backfill already modified the original tokens array so that future passes would detect the token as T_FN, but the return type check needs to process the T_FN token itself, so continue processing and let the T_FN token get added as normal.
squizlabs · Jan 21, 2020 · a4d4cc3 · a4d4cc3
1 parent 0e4fe74
commit a4d4cc3
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 27 deletions.
diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
@@ -1208,6 +1208,21 @@ protected function tokenize($string)
                 continue;
             }
 
+            /*
+                Backfill the T_FN token for PHP versions < 7.4.
+            */
+
+            if ($tokenIsArray === true
+                && $token[0] === T_STRING
+                && strtolower($token[1]) === 'fn'
+            ) {
+                // Modify the original token stack so that
+                // future checks (like looking for T_NULLABLE) can
+                // detect the T_FN token more easily.
+                $tokens[$stackPtr][0] = T_FN;
+                $token[0] = T_FN;
+            }
+
             /*
                 The string-like token after a function keyword should always be
                 tokenized as T_STRING even if it appears to be a different token,
@@ -1357,33 +1372,6 @@ function return types. We want to keep the parenthesis map clean,
                 continue;
             }
 
-            /*
-                Backfill the T_FN token for PHP versions < 7.4.
-            */
-
-            if ($tokenIsArray === true
-                && $token[0] === T_STRING
-                && strtolower($token[1]) === 'fn'
-            ) {
-                // Modify the original token stack so that
-                // future checks (like looking for T_NULLABLE) can
-                // detect the T_FN token more easily.
-                $tokens[$stackPtr][0] = T_FN;
-
-                $finalTokens[$newStackPtr] = [
-                    'content' => $token[1],
-                    'code'    => T_FN,
-                    'type'    => 'T_FN',
-                ];
-
-                if (PHP_CODESNIFFER_VERBOSITY > 1) {
-                    echo "\t\t* token $stackPtr changed from T_STRING to T_FN".PHP_EOL;
-                }
-
-                $newStackPtr++;
-                continue;
-            }
-
             /*
                 PHP doesn't assign a token to goto labels, so we have to.
                 These are just string tokens with a single colon after them. Double

diff --git a/tests/Core/Tokenizer/BackfillFnTokenTest.inc b/tests/Core/Tokenizer/BackfillFnTokenTest.inc
@@ -67,5 +67,8 @@ fn(parent $a) : parent => $a;
 /* testCallableReturnType */
 fn(callable $a) : callable => $a;
 
+/* testArrayReturnType */
+fn(array $a) : array => $a;
+
 /* testTernary */
 $fn = fn($a) => $a ? /* testTernaryThen */ fn() : string => 'a' : /* testTernaryElse */ fn() : string => 'b';
diff --git a/tests/Core/Tokenizer/BackfillFnTokenTest.php b/tests/Core/Tokenizer/BackfillFnTokenTest.php
@@ -474,6 +474,7 @@ public function testKeywordReturnTypes()
             'Self',
             'Parent',
             'Callable',
+            'Array',
         ];
 
         foreach ($testMarkers as $marker) {