Tokenizer/PHP: bug fix in improved context sensitive keyword support [2]

While investigating 3607, I figured adding a test with a function declared to return by reference would also not be amiss and found that that situation was not accounted for. This commit fixes that. Includes unit test.
squizlabs · Jun 12, 2022 · d9cf568 · d9cf568
1 parent 854aba7
commit d9cf568
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 22 deletions.
diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
@@ -607,38 +607,57 @@ protected function tokenize($string)
 
             if ($tokenIsArray === true
                 && isset(Util\Tokens::$contextSensitiveKeywords[$token[0]]) === true
-                && isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
+                && (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
+                || $finalTokens[$lastNotEmptyToken]['content'] === '&')
             ) {
-                $preserveKeyword = false;
+                if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
+                    $preserveKeyword = false;
 
-                // `new class`, and `new static` should be preserved.
-                if ($finalTokens[$lastNotEmptyToken]['code'] === T_NEW
-                    && ($token[0] === T_CLASS
-                    || $token[0] === T_STATIC)
-                ) {
-                    $preserveKeyword = true;
-                }
+                    // `new class`, and `new static` should be preserved.
+                    if ($finalTokens[$lastNotEmptyToken]['code'] === T_NEW
+                        && ($token[0] === T_CLASS
+                        || $token[0] === T_STATIC)
+                    ) {
+                        $preserveKeyword = true;
+                    }
 
-                // `new class extends` `new class implements` should be preserved
-                if (($token[0] === T_EXTENDS || $token[0] === T_IMPLEMENTS)
-                    && $finalTokens[$lastNotEmptyToken]['code'] === T_CLASS
-                ) {
-                    $preserveKeyword = true;
-                }
+                    // `new class extends` `new class implements` should be preserved
+                    if (($token[0] === T_EXTENDS || $token[0] === T_IMPLEMENTS)
+                        && $finalTokens[$lastNotEmptyToken]['code'] === T_CLASS
+                    ) {
+                        $preserveKeyword = true;
+                    }
+
+                    // `namespace\` should be preserved
+                    if ($token[0] === T_NAMESPACE) {
+                        for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
+                            if (is_array($tokens[$i]) === false) {
+                                break;
+                            }
+
+                            if (isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true) {
+                                continue;
+                            }
+
+                            if ($tokens[$i][0] === T_NS_SEPARATOR) {
+                                $preserveKeyword = true;
+                            }
 
-                // `namespace\` should be preserved
-                if ($token[0] === T_NAMESPACE) {
-                    for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
-                        if (is_array($tokens[$i]) === false) {
                             break;
                         }
+                    }
+                }//end if
+
+                if ($finalTokens[$lastNotEmptyToken]['content'] === '&') {
+                    $preserveKeyword = true;
 
-                        if (isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true) {
+                    for ($i = ($lastNotEmptyToken - 1); $i >= 0; $i--) {
+                        if (isset(Util\Tokens::$emptyTokens[$finalTokens[$i]['code']]) === true) {
                             continue;
                         }
 
-                        if ($tokens[$i][0] === T_NS_SEPARATOR) {
-                            $preserveKeyword = true;
+                        if ($finalTokens[$i]['code'] === T_FUNCTION) {
+                            $preserveKeyword = false;
                         }
 
                         break;

diff --git a/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc b/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc
@@ -217,3 +217,4 @@ class Foo extends /* testNamespaceInNameIsKeyword */ namespace\Exception
 {}
 
 function /* testKeywordAfterFunctionShouldBeString */ eval() {}
+function /* testKeywordAfterFunctionByRefShouldBeString */ &switch() {}
diff --git a/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php b/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php
@@ -124,6 +124,7 @@ public function dataStrings()
             ['/* testNamespaceNameIsString3 */'],
 
             ['/* testKeywordAfterFunctionShouldBeString */'],
+            ['/* testKeywordAfterFunctionByRefShouldBeString */'],
         ];
 
     }//end dataStrings()
Original file line number	Diff line number	Diff line change
Expand Up		@@ -217,3 +217,4 @@ class Foo extends /* testNamespaceInNameIsKeyword */ namespace\Exception
		{}

		function /* testKeywordAfterFunctionShouldBeString */ eval() {}
		function /* testKeywordAfterFunctionByRefShouldBeString */ &switch() {}