PHP Tokenizer: improve arrow function backfill consistency

This commit implements the proposal as outlined in 2859#issuecomment-583695917.

The `fn` keyword will now only be tokenized as `T_FN` if it really is an arrow function, in which case it will also have the additional indexes set in the token array.

In all other cases - including when the keyword is used in a function declaration for a named function or method -, it will be tokenized as `T_STRING`.

Includes numerous additional unit tests.

Includes removing the changes made to the `File::getDeclarationName()` function and the `AbstractPatternSniff` in commit 37dda44 as those are no longer necessary.

Fixes #2859

src/Files/File.php

@@ -1254,9 +1254,7 @@ public function getDeclarationName($stackPtr)
 
         $content = null;
         for ($i = $stackPtr; $i < $this->numTokens; $i++) {
-            if ($this->tokens[$i]['code'] === T_STRING
-                || $this->tokens[$i]['code'] === T_FN
-            ) {
+            if ($this->tokens[$i]['code'] === T_STRING) {
                 $content = $this->tokens[$i]['content'];
                 break;
             }

src/Sniffs/AbstractPatternSniff.php

@@ -614,9 +614,7 @@ protected function processPattern($patternInfo, File $phpcsFile, $stackPtr)
                     $stackPtr = ($tokens[$next][$pattern[$i]['to']] + 1);
                 }//end if
             } else if ($pattern[$i]['type'] === 'string') {
-                if ($tokens[$stackPtr]['code'] !== T_STRING
-                    && $tokens[$stackPtr]['code'] !== T_FN
-                ) {
+                if ($tokens[$stackPtr]['code'] !== T_STRING) {
                     $hasError = true;
                 }
 

src/Tokenizers/PHP.php

@@ -1894,7 +1894,7 @@ protected function processAdditional()
                             $this->tokens[$i]['scope_closer']    = $scopeCloser;
                             $this->tokens[$i]['parenthesis_owner']  = $i;
                             $this->tokens[$i]['parenthesis_opener'] = $x;
-                            $this->tokens[$i]['parenthesis_closer'] = $this->tokens[$x]['parenthesis_closer'];
+                            $this->tokens[$i]['parenthesis_closer'] = $closer;
 
                             $this->tokens[$arrow]['code'] = T_FN_ARROW;
                             $this->tokens[$arrow]['type'] = 'T_FN_ARROW';
@@ -1918,6 +1918,12 @@ protected function processAdditional()
                         }//end if
                     }//end if
                 }//end if
+
+                // If after all that, the extra tokens are not set, this is not an arrow function.
+                if (isset($this->tokens[$i]['scope_closer']) === false) {
+                    $this->tokens[$i]['code'] = T_STRING;
+                    $this->tokens[$i]['type'] = 'T_STRING';
+                }
             } else if ($this->tokens[$i]['code'] === T_OPEN_SQUARE_BRACKET) {
                 if (isset($this->tokens[$i]['bracket_closer']) === false) {
                     continue;

tests/Core/Tokenizer/BackfillFnTokenTest.inc

@@ -73,6 +73,49 @@ fn(array $a) : array => $a;
 /* testTernary */
 $fn = fn($a) => $a ? /* testTernaryThen */ fn() : string => 'a' : /* testTernaryElse */ fn() : string => 'b';
 
+/* testConstantDeclaration */
+const FN = 'a';
+
+class Foo {
+    /* testStaticMethodName */
+    public static function fn($param) {
+        /* testNestedInMethod */
+        $fn = fn($c) => $callable($factory($c), $c);
+    }
+
+    public function foo() {
+        /* testPropertyAssignment */
+        $this->fn = 'a';
+    }
+}
+
+$anon = new class() {
+    /* testAnonClassMethodName */
+    protected function fn($param) {
+    }
+}
+
+/* testNonArrowStaticMethodCall */
+$a = Foo::fn($param);
+
+/* testNonArrowStaticMethodCallWithChaining */
+$a = Foo::fn($param)->another();
+
+/* testNonArrowStaticConstant */
+$a = MyClass::FN;
+
+/* testNonArrowStaticConstantDeref */
+$a = MyClass::FN[$a];
+
+/* testNonArrowObjectMethodCall */
+$a = $obj->fn($param);
+
+/* testNonArrowNamespacedFunctionCall */
+$a = MyNS\Sub\fn($param);
+
+/* testNonArrowNamespaceOperatorFunctionCall */
+$a = namespace\fn($param);
+
 /* testLiveCoding */
 // Intentional parse error. This has to be the last test in the file.
 $fn = fn

tests/Core/Tokenizer/BackfillFnTokenTest.php

@@ -101,28 +101,6 @@ public function testComments()
     }//end testComments()
 
 
-    /**
-     * Test a function called fn.
-     *
-     * @covers PHP_CodeSniffer\Tokenizers\PHP::processAdditional
-     *
-     * @return void
-     */
-    public function testFunctionName()
-    {
-        $tokens = self::$phpcsFile->getTokens();
-
-        $token = $this->getTargetToken('/* testFunctionName */', T_FN);
-        $this->assertFalse(array_key_exists('scope_condition', $tokens[$token]), 'Scope condition is set');
-        $this->assertFalse(array_key_exists('scope_opener', $tokens[$token]), 'Scope opener is set');
-        $this->assertFalse(array_key_exists('scope_closer', $tokens[$token]), 'Scope closer is set');
-        $this->assertFalse(array_key_exists('parenthesis_owner', $tokens[$token]), 'Parenthesis owner is set');
-        $this->assertFalse(array_key_exists('parenthesis_opener', $tokens[$token]), 'Parenthesis opener is set');
-        $this->assertFalse(array_key_exists('parenthesis_closer', $tokens[$token]), 'Parenthesis closer is set');
-
-    }//end testFunctionName()
-
-
     /**
      * Test nested arrow functions.
      *
@@ -553,27 +531,85 @@ public function testTernary()
 
 
     /**
-     * Test that the backfill presumes T_FN during live coding, but doesn't set the additional index keys.
+     * Test arrow function nested within a method declaration.
      *
      * @covers PHP_CodeSniffer\Tokenizers\PHP::processAdditional
      *
      * @return void
      */
-    public function testLiveCoding()
+    public function testNestedInMethod()
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $token = $this->getTargetToken('/* testNestedInMethod */', T_FN);
+        $this->backfillHelper($token);
+
+        $this->assertSame($tokens[$token]['scope_opener'], ($token + 5), 'Scope opener is not the arrow token');
+        $this->assertSame($tokens[$token]['scope_closer'], ($token + 17), 'Scope closer is not the semicolon token');
+
+        $opener = $tokens[$token]['scope_opener'];
+        $this->assertSame($tokens[$opener]['scope_opener'], ($token + 5), 'Opener scope opener is not the arrow token');
+        $this->assertSame($tokens[$opener]['scope_closer'], ($token + 17), 'Opener scope closer is not the semicolon token');
+
+        $closer = $tokens[$token]['scope_opener'];
+        $this->assertSame($tokens[$closer]['scope_opener'], ($token + 5), 'Closer scope opener is not the arrow token');
+        $this->assertSame($tokens[$closer]['scope_closer'], ($token + 17), 'Closer scope closer is not the semicolon token');
+
+    }//end testNestedInMethod()
+
+
+    /**
+     * Verify that "fn" keywords which are not arrow functions get tokenized as T_STRING and don't
+     * have the extra token array indexes.
+     *
+     * @param string $testMarker The comment prefacing the target token.
+     *
+     * @dataProvider dataNotAnArrowFunction
+     * @covers       PHP_CodeSniffer\Tokenizers\PHP::processAdditional
+     *
+     * @return void
+     */
+    public function testNotAnArrowFunction($testMarker)
     {
         $tokens = self::$phpcsFile->getTokens();
 
-        $token = $this->getTargetToken('/* testLiveCoding */', [T_STRING, T_FN]);
-        $this->assertSame($tokens[$token]['code'], T_FN, 'Token not tokenized as T_FN');
+        $token      = $this->getTargetToken('/* testFunctionName */', [T_STRING, T_FN], 'fn');
+        $tokenArray = $tokens[$token];
+
+        $this->assertSame('T_STRING', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING');
+
+        $this->assertArrayNotHasKey('scope_condition', $tokenArray, 'Scope condition is set');
+        $this->assertArrayNotHasKey('scope_opener', $tokenArray, 'Scope opener is set');
+        $this->assertArrayNotHasKey('scope_closer', $tokenArray, 'Scope closer is set');
+        $this->assertArrayNotHasKey('parenthesis_owner', $tokenArray, 'Parenthesis owner is set');
+        $this->assertArrayNotHasKey('parenthesis_opener', $tokenArray, 'Parenthesis opener is set');
+        $this->assertArrayNotHasKey('parenthesis_closer', $tokenArray, 'Parenthesis closer is set');
 
-        $this->assertArrayNotHasKey('scope_condition', $tokens[$token], 'Scope condition is set');
-        $this->assertArrayNotHasKey('scope_opener', $tokens[$token], 'Scope opener is set');
-        $this->assertArrayNotHasKey('scope_closer', $tokens[$token], 'Scope closer is set');
-        $this->assertArrayNotHasKey('parenthesis_owner', $tokens[$token], 'Parenthesis owner is set');
-        $this->assertArrayNotHasKey('parenthesis_opener', $tokens[$token], 'Parenthesis opener is set');
-        $this->assertArrayNotHasKey('parenthesis_closer', $tokens[$token], 'Parenthesis closer is set');
+    }//end testNotAnArrowFunction()
+
+
+    /**
+     * Data provider.
+     *
+     * @see testNotAnArrowFunction()
+     *
+     * @return array
+     */
+    public function dataNotAnArrowFunction()
+    {
+        return [
+            ['/* testFunctionName */'],
+            ['/* testStaticMethodName */'],
+            ['/* testAnonClassMethodName */'],
+            ['/* testNonArrowStaticMethodCall */'],
+            ['/* testNonArrowStaticMethodCallWithChaining */'],
+            ['/* testNonArrowObjectMethodCall */'],
+            ['/* testNonArrowNamespacedFunctionCall */'],
+            ['/* testNonArrowNamespaceOperatorFunctionCall */'],
+            ['/* testLiveCoding */'],
+        ];
 
-    }//end testLiveCoding()
+    }//end dataNotAnArrowFunction()
 
 
     /**