Arrays::isShortArray()/Lists::isShortList(): bugfix for tokenizer issue icw short open tag

PHPCSUtils/Utils/Arrays.php

@@ -105,7 +105,9 @@ public static function isShortArray(File $phpcsFile, $stackPtr)
                  *
                  * @link https://github.com/squizlabs/PHP_CodeSniffer/issues/1971
                  */
-                if ($prevNonEmpty !== 0 || $tokens[$prevNonEmpty]['code'] !== \T_OPEN_TAG) {
+                if ($prevNonEmpty !== 0
+                    || isset(Collections::phpOpenTags()[$tokens[$prevNonEmpty]['code']]) === false
+                ) {
                     return false;
                 }
             }

PHPCSUtils/Utils/Lists.php

@@ -106,7 +106,8 @@ public static function isShortList(File $phpcsFile, $stackPtr)
             }
 
             $prevNonEmpty = $phpcsFile->findPrevious(Tokens::$emptyTokens, ($opener - 1), null, true);
-            if ((($prevNonEmpty === 0 && $tokens[$prevNonEmpty]['code'] === \T_OPEN_TAG) // Bug #1971.
+            if ((($prevNonEmpty === 0
+                && isset(Collections::phpOpenTags()[$tokens[$prevNonEmpty]['code']]) === true) // Bug #1971.
                 || ($tokens[$prevNonEmpty]['code'] === \T_CLOSE_CURLY_BRACKET
                     && isset($tokens[$prevNonEmpty]['scope_condition']))) // Bug #1284.
             ) {

Tests/Utils/Lists/IsShortArrayOrListTokenizerBC3Test.inc

@@ -0,0 +1,5 @@
+<?=
+
+/* testTokenizerIssue1971PHPCSlt330gt271E */
+// Valid PHP, though not useful as the implied echo will break on the array.
+[1, 2, /* testTokenizerIssue1971PHPCSlt330gt271F */ [3], 4];

Tests/Utils/Lists/IsShortArrayOrListTokenizerBC3Test.php

@@ -0,0 +1,119 @@
+<?php
+/**
+ * PHPCSUtils, utility functions and classes for PHP_CodeSniffer sniff developers.
+ *
+ * @package   PHPCSUtils
+ * @copyright 2019-2020 PHPCSUtils Contributors
+ * @license   https://opensource.org/licenses/LGPL-3.0 LGPL3
+ * @link      https://github.com/PHPCSStandards/PHPCSUtils
+ */
+
+namespace PHPCSUtils\Tests\Utils\Lists;
+
+use PHPCSUtils\TestUtils\UtilityMethodTestCase;
+use PHPCSUtils\Utils\Arrays;
+use PHPCSUtils\Utils\Lists;
+
+/**
+ * Tests for specific PHPCS tokenizer issues which can affect the \PHPCSUtils\Utils\Arrays::isShortArray()
+ * and the \PHPCSUtils\Utils\Lists::isShortList() methods.
+ *
+ * @group arrays
+ * @group lists
+ *
+ * @since 1.0.0
+ */
+class IsShortArrayOrListTokenizerBC3Test extends UtilityMethodTestCase
+{
+
+    /**
+     * Data integrity test. Verify that the data provider is consistent.
+     *
+     * Something is either a short array or a short list or neither. It can never be both at the same time.
+     *
+     * @dataProvider  dataIsShortArrayOrList
+     * @coversNothing
+     *
+     * @param string $ignore Unused.
+     * @param bool[] $data   The expected boolean return value for list and array.
+     *
+     * @return void
+     */
+    public function testValidDataProvider($ignore, $data)
+    {
+        $forbidden = [
+            'list'  => true,
+            'array' => true,
+        ];
+
+        $this->assertNotSame($forbidden, $data);
+    }
+
+    /**
+     * Test correctly determining whether a short array open token is a short array,
+     * even when the token is incorrectly tokenized.
+     *
+     * @dataProvider dataIsShortArrayOrList
+     * @covers       \PHPCSUtils\Utils\Arrays::isShortArray
+     *
+     * @param string $testMarker The comment which prefaces the target token in the test file.
+     * @param bool[] $expected   The expected boolean return value for list and array.
+     *
+     * @return void
+     */
+    public function testIsShortArray($testMarker, $expected)
+    {
+        $stackPtr = $this->getTargetToken($testMarker, [\T_OPEN_SHORT_ARRAY, \T_OPEN_SQUARE_BRACKET]);
+        $result   = Arrays::isShortArray(self::$phpcsFile, $stackPtr);
+
+        $this->assertSame($expected['array'], $result);
+    }
+
+    /**
+     * Test correctly determining whether a short array open token is a short array or a short list,
+     * even when the token is incorrectly tokenized.
+     *
+     * @dataProvider dataIsShortArrayOrList
+     * @covers       \PHPCSUtils\Utils\Lists::isShortList
+     *
+     * @param string $testMarker The comment which prefaces the target token in the test file.
+     * @param bool[] $expected   The expected boolean return value for list and array.
+     *
+     * @return void
+     */
+    public function testIsShortList($testMarker, $expected)
+    {
+        $stackPtr = $this->getTargetToken($testMarker, [\T_OPEN_SHORT_ARRAY, \T_OPEN_SQUARE_BRACKET]);
+        $result   = Lists::isShortList(self::$phpcsFile, $stackPtr);
+
+        $this->assertSame($expected['list'], $result);
+    }
+
+    /**
+     * Data provider.
+     *
+     * @see testIsShortArray() For the array format.
+     * @see testIsShortList()  For the array format.
+     *
+     * @return array
+     */
+    public function dataIsShortArrayOrList()
+    {
+        return [
+            'issue-1971-short-array-first-in-file' => [
+                '/* testTokenizerIssue1971PHPCSlt330gt271E */',
+                [
+                    'array' => true,
+                    'list'  => false,
+                ],
+            ],
+            'issue-1971-short-array-first-in-file-nested' => [
+                '/* testTokenizerIssue1971PHPCSlt330gt271F */',
+                [
+                    'array' => true,
+                    'list'  => false,
+                ],
+            ],
+        ];
+    }
+}