Skip to content

Commit

Permalink
PHP 7.4 numeric separators are now tokenized in the same way when usi…
Browse files Browse the repository at this point in the history 
…ng older PHP versions (ref #2546)
  • Loading branch information
@gsherwood
gsherwood committed Nov 19, 2019
1 parent eb378ac commit 119a702
Show file tree
Hide file tree
Showing 4 changed files with 193 additions and 0 deletions.
10 changes: 10 additions & 0 deletions package.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ http://pear.php.net/dtd/package-2.0.xsd">
-- The token after the statement (normally a semicolon) becomes the scope closer
-- The token is also associated with the opening and closing parenthesis of the statement
-- Any functions named "fn" will cause have a T_FN token for the function name, but have no scope information
- PHP 7.4 numeric separators are now tokenized in the same way when using older PHP versions
-- Previously, a number like 1_000 would tokenize as T_LNUMBER (1), T_STRING (_000)
-- Now, the number tokenizes as T_LNUMBER (1_000)
-- Sniff developers should consider how numbers with underscores impact their custom sniffs
- The PHPCS file cache now takes file permissions into account
-- The cache is now invalidated for a file when its permissions are changed
- File::getMethodParameters() now supports arrow functions
Expand Down Expand Up @@ -125,6 +129,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<file baseinstalldir="" name="AnonClassParenthesisOwnerTest.php" role="test" />
<file baseinstalldir="" name="BackfillFnTokenTest.inc" role="test" />
<file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
<file baseinstalldir="" name="BackfillNumericSeparatorTest.inc" role="test" />
<file baseinstalldir="" name="BackfillNumericSeparatorTest.php" role="test" />
</dir>
<file baseinstalldir="" name="AbstractMethodUnitTest.php" role="test" />
<file baseinstalldir="" name="AllTests.php" role="test" />
Expand Down Expand Up @@ -1960,6 +1966,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
</filelist>
Expand Down Expand Up @@ -2001,6 +2009,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
<ignore name="bin/phpcs.bat" />
Expand Down
60 changes: 60 additions & 0 deletions src/Tokenizers/PHP.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -968,6 +968,66 @@ protected function tokenize($string)
continue;
}

/*
Before PHP 7.4, underscores inside T_LNUMBER and T_DNUMBER
tokens split the token with a T_STRING. So look for
and change these tokens in earlier versions.
*/

if ($tokenIsArray === true
&& ($token[0] === T_LNUMBER
|| $token[0] === T_DNUMBER)
&& isset($tokens[($stackPtr + 1)]) === true
&& is_array($tokens[($stackPtr + 1)]) === true
&& $tokens[($stackPtr + 1)][0] === T_STRING
&& $tokens[($stackPtr + 1)][1][0] === '_'
) {
$newContent = $token[1];
$newType = $token[0];
for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
if (is_array($tokens[$i]) === false) {
break;
}

if ($tokens[$i][0] === T_LNUMBER
|| $tokens[$i][0] === T_DNUMBER
|| ($tokens[$i][0] === T_STRING
&& $tokens[$i][1][0] === '_')
) {
$newContent .= $tokens[$i][1];

// Any T_DNUMBER token needs to make the
// new number a T_DNUMBER as well.
if ($tokens[$i][0] === T_DNUMBER) {
$newType = T_DNUMBER;
}

// Support floats.
if ($tokens[$i][0] === T_STRING
&& substr(strtolower($tokens[$i][1]), -1) === 'e'
&& $tokens[($i + 1)] === '-'
) {
$newContent .= '-';
$i++;
}

continue;
}

break;
}//end for

$newToken = [];
$newToken['code'] = $newType;
$newToken['type'] = Util\Tokens::tokenName($token[0]);
$newToken['content'] = $newContent;
$finalTokens[$newStackPtr] = $newToken;

$newStackPtr++;
$stackPtr = ($i - 1);
continue;
}//end if

/*
Convert ? to T_NULLABLE OR T_INLINE_THEN
*/
Expand Down
22 changes: 22 additions & 0 deletions tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<?php

/* testSimpleLNumber */
$foo = 1_000_000_000;

/* testSimpleDNumber */
‪$foo = 107_925_284.88;

/* testFloat */
$foo = 6.674_083e-11;

/* testHex */
$foo = 0xCAFE_F00D;

/* testHexMultiple */
$foo = 0x42_72_6F_77_6E;

/* testBinary */
$foo = 0b0101_1111;

/* testOctal */
$foo = 0137_041;
101 changes: 101 additions & 0 deletions tests/Core/Tokenizer/BackfillNumericSeparatorTest.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
<?php
/**
* Tests the backfilling of numeric seperators to PHP < 7.4.
*
* @author Greg Sherwood <gsherwood@squiz.net>
* @copyright 2019 Squiz Pty Ltd (ABN 77 084 670 600)
* @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
*/

namespace PHP_CodeSniffer\Tests\Core\Tokenizer;

use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest;

class BackfillNumericSeparatorTest extends AbstractMethodUnitTest
{


/**
* Test that numbers using numeric seperators are tokenized correctly.
*
* @param array $testData The data required for the specific test case.
*
* @dataProvider dataTestBackfill
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
*
* @return void
*/
public function testBackfill($testData)
{
$tokens = self::$phpcsFile->getTokens();
$number = $this->getTargetToken($testData['marker'], $testData['type']);
$this->assertSame($tokens[$number]['content'], $testData['value']);

}//end testBackfill()


/**
* Data provider.
*
* @see testBackfill()
*
* @return array
*/
public function dataTestBackfill()
{
return [
[
[
'marker' => '/* testSimpleLNumber */',
'type' => T_LNUMBER,
'value' => '1_000_000_000',
],
],
[
[
'marker' => '/* testSimpleDNumber */',
'type' => T_DNUMBER,
'value' => '107_925_284.88',
],
],
[
[
'marker' => '/* testFloat */',
'type' => T_DNUMBER,
'value' => '6.674_083e-11',
],
],
[
[
'marker' => '/* testHex */',
'type' => T_LNUMBER,
'value' => '0xCAFE_F00D',
],
],
[
[
'marker' => '/* testHexMultiple */',
'type' => T_LNUMBER,
'value' => '0x42_72_6F_77_6E',
],
],
[
[
'marker' => '/* testBinary */',
'type' => T_LNUMBER,
'value' => '0b0101_1111',
],
],
[
[
'marker' => '/* testOctal */',
'type' => T_LNUMBER,
'value' => '0137_041',
],
],
];

}//end dataTestBackfill()


}//end class

0 comments on commit 119a702

Please sign in to comment.