Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Migrate the PHP Compactor to the new PHPToken API #619

Merged
merged 1 commit into from
Feb 20, 2022
Merged

Migrate the PHP Compactor to the new PHPToken API #619

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 58 additions & 54 deletions src/Compactor/Php.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,19 @@
use function array_slice;
use function array_splice;
use function count;
use function in_array;
use function is_array;
use function is_int;
use function is_string;
use KevinGH\Box\Annotation\DocblockAnnotationParser;
use KevinGH\Box\Annotation\InvalidToken;
use function ltrim;
use PhpToken;
use function preg_replace;
use RuntimeException;
use function str_repeat;
use function strpos;
use function substr;
use function substr_count;
use const T_COMMENT;
use const T_DOC_COMMENT;
use const T_WHITESPACE;
use function token_get_all;
use Webmozart\Assert\Assert;

/**
* A PHP source code compactor copied from Composer.
Expand All @@ -51,7 +47,7 @@
*/
final class Php extends FileExtensionCompactor
{
private $annotationParser;
private DocblockAnnotationParser $annotationParser;

public function __construct(DocblockAnnotationParser $annotationParser, array $extensions = ['php'])
{
Expand All @@ -63,56 +59,52 @@ public function __construct(DocblockAnnotationParser $annotationParser, array $e
protected function compactContent(string $contents): string
{
$output = '';
$tokens = token_get_all($contents);
$tokens = PhpToken::tokenize($contents);
$tokenCount = count($tokens);

for ($index = 0; $index < $tokenCount; ++$index) {
$token = $tokens[$index];
if (is_string($token)) {
$output .= $token;
} elseif (in_array($token[0], [T_COMMENT, T_DOC_COMMENT], true)) {
if (0 === strpos($token[1], '#[')) {
// This is, in all likelyhood, the start of a PHP >= 8.0 attribute.
$tokenText = $token->text;

if ($token->is([T_COMMENT, T_DOC_COMMENT])) {
if (str_starts_with($tokenText, '#[')) {
// This is, in all likelihood, the start of a PHP >= 8.0 attribute.
// Note: $tokens may be updated by reference as well!
$retokenized = $this->retokenizeAttribute($tokens, $index);

if (null !== $retokenized) {
array_splice($tokens, $index, 1, $retokenized);
$tokenCount = count($tokens);
}

$attributeCloser = $this->findAttributeCloser($tokens, $index);
$attributeCloser = self::findAttributeCloser($tokens, $index);

if (is_int($attributeCloser)) {
$output .= '#[';
} else {
// Turns out this was not an attribute. Treat it as a plain comment.
$output .= str_repeat("\n", substr_count($token[1], "\n"));
$output .= str_repeat("\n", substr_count($tokenText, "\n"));
}
} elseif (false !== strpos($token[1], '@')) {
} elseif (str_contains($tokenText, '@')) {
try {
$output .= $this->compactAnnotations($token[1]);
} catch (InvalidToken $exception) {
// This exception is due to the dumper to be out of sync with the current grammar and/or the
// grammar being incomplete. In both cases throwing here is better in order to identify and
// this those cases instead of silently failing.

throw $exception;
} catch (RuntimeException $exception) {
$output .= $token[1];
$output .= $this->compactAnnotations($tokenText);
} catch (RuntimeException) {
$output .= $tokenText;
}
} else {
$output .= str_repeat("\n", substr_count($token[1], "\n"));
$output .= str_repeat("\n", substr_count($tokenText, "\n"));
}
} elseif (T_WHITESPACE === $token[0]) {
$whitespace = $token[1];
} elseif ($token->is(T_WHITESPACE)) {
$whitespace = $tokenText;
$previousIndex = ($index - 1);

// Handle whitespace potentially being split into two tokens after attribute retokenization.
if (isset($tokens[$index + 1])
&& is_array($tokens[$index + 1])
&& T_WHITESPACE === $tokens[$index + 1][0]
$nextToken = $tokens[$index + 1] ?? null;

if (null !== $nextToken
&& $nextToken->is(T_WHITESPACE)
) {
$whitespace .= $tokens[$index + 1][1];
$whitespace .= $nextToken->text;
++$index;
}

Expand All @@ -124,9 +116,11 @@ protected function compactContent(string $contents): string

// If the new line was split off from the whitespace token due to it being included in
// the previous (comment) token (PHP < 8), remove leading spaces.
if (is_array($tokens[$previousIndex])
&& T_COMMENT === $tokens[$previousIndex][0]
&& false !== strpos($tokens[$previousIndex][1], "\n")

$previousToken = $tokens[$previousIndex];

if ($previousToken->is(T_COMMENT)
&& str_contains($previousToken->text, "\n")
) {
$whitespace = ltrim($whitespace, ' ');
}
Expand All @@ -136,7 +130,7 @@ protected function compactContent(string $contents): string

$output .= $whitespace;
} else {
$output .= $token[1];
$output .= $tokenText;
}
}

Expand Down Expand Up @@ -172,27 +166,29 @@ private function compactAnnotations(string $docblock): string
return $compactedDocblock;
}

private function findAttributeCloser(array $tokens, int $opener): ?int
/**
* @param list<PhpToken> $tokens
*/
private static function findAttributeCloser(array $tokens, int $opener): ?int
{
$tokenCount = count($tokens);
$brackets = [$opener];
$closer = null;

for ($i = ($opener + 1); $i < $tokenCount; ++$i) {
if (false === is_string($tokens[$i])) {
continue;
}
$tokenText = $tokens[$i]->text;

// Allow for short arrays within attributes.
if ('[' === $tokens[$i]) {
if ('[' === $tokenText) {
$brackets[] = $i;

continue;
}

if (']' === $tokens[$i]) {
if (']' === $tokenText) {
array_pop($brackets);
if (empty($brackets)) {

if (0 === count($brackets)) {
$closer = $i;
break;
}
Expand All @@ -202,35 +198,43 @@ private function findAttributeCloser(array $tokens, int $opener): ?int
return $closer;
}

/**
* @param non-empty-list<PhpToken> $tokens
*/
private function retokenizeAttribute(array &$tokens, int $opener): ?array
{
Assert::keyExists($tokens, $opener);

/** @var PhpToken $token */
$token = $tokens[$opener];
$attributeBody = substr($token[1], 2);
$subTokens = @token_get_all('<?php '.$attributeBody);
$attributeBody = substr($token->text, 2);
$subTokens = PhpToken::tokenize('<?php '.$attributeBody);

// Replace the PHP open tag with the attribute opener as a simple token.
array_splice($subTokens, 0, 1, ['#[']);

$closer = $this->findAttributeCloser($subTokens, 0);
$closer = self::findAttributeCloser($subTokens, 0);

// Multi-line attribute or attribute containing something which looks like a PHP close tag.
// Retokenize the rest of the file after the attribute opener.
if (null === $closer) {
foreach (array_slice($tokens, ($opener + 1)) as $token) {
if (is_array($token)) {
$attributeBody .= $token[1];
} else {
$attributeBody .= $token;
}
$attributeBody .= $token->text;
}

$subTokens = @token_get_all('<?php '.$attributeBody);
$subTokens = PhpToken::tokenize('<?php '.$attributeBody);
array_splice($subTokens, 0, 1, ['#[']);

$closer = $this->findAttributeCloser($subTokens, 0);
$closer = self::findAttributeCloser($subTokens, 0);

if (null !== $closer) {
array_splice($tokens, ($opener + 1), count($tokens), array_slice($subTokens, ($closer + 1)));
array_splice(
$tokens,
($opener + 1),
count($tokens),
array_slice($subTokens, ($closer + 1)),
);

$subTokens = array_slice($subTokens, 0, ($closer + 1));
}
}
Expand Down