Work on properly parsing the expression lists passed to <?=

Before, tolerant-php-parser parsed the tokens after `<?=` and `<?php` the same way. Fixes microsoft#220 (This commit and subsequent commits in this PR) This adds `TokenKind::ScriptSectionStartWithEchoTag` and handles the subsequent tokens differently if that token kind is seen instead of `ScriptSectionStartTag`. This reuses `EchoExpression` for simplicity. The resulting expression will have no `echoKeyword` because `<?=` was part of the preceding `InlineHTML` Node. - In the majority of cases, the EchoExpression will be moved into the outer statement list.
TysonAndre · May 13, 2018 · d0ab7a4 · d0ab7a4
1 parent 7647b60
commit d0ab7a4
Show file tree

Hide file tree

Showing 9 changed files with 44 additions and 10 deletions.
diff --git a/src/Node/Expression/EchoExpression.php b/src/Node/Expression/EchoExpression.php
@@ -12,7 +12,7 @@
 
 class EchoExpression extends Expression {
 
-    /** @var Token */
+    /** @var Token|null this can be null if generated from <?= */
     public $echoKeyword;
 
     /** @var ExpressionList */

diff --git a/src/Node/Statement/InlineHtml.php b/src/Node/Statement/InlineHtml.php
@@ -19,9 +19,21 @@ class InlineHtml extends StatementNode {
     /** @var Token|null */
     public $scriptSectionStartTag;
 
+    /**
+     * @var EchoExpression|null used to represent <?= while parsing.
+     * This may be non-null in the final AST.
+     *
+     * This will be null under any of these conditions:
+     *
+     * - The scriptSectionStartTag isn't TokenKind::ScriptSectionStartWithEchoTag,
+     * - The echoExpression was normalized and moved into a statement list
+     */
+    public $echoExpression;
+
     const CHILD_NAMES = [
         'scriptSectionEndTag',
         'text',
-        'scriptSectionStartTag'
+        'scriptSectionStartTag',
+        'echoExpression',
     ];
 }
diff --git a/src/Parser.php b/src/Parser.php
@@ -152,7 +152,12 @@ public function parseSourceFile(string $fileContents, string $uri = null) : Sour
         $sourceFile->uri = $uri;
         $sourceFile->statementList = array();
         if ($this->getCurrentToken()->kind !== TokenKind::EndOfFileToken) {
-            $sourceFile->statementList[] = $this->parseInlineHtml($sourceFile);
+            $inlineHTML = $this->parseInlineHtml($sourceFile);
+            $sourceFile->statementList[] = $inlineHTML;
+            if ($inlineHTML->echoExpression) {
+                $sourceFile[] = $inlineHTML->echoExpression;
+                $inlineHTML->echoExpression = null;
+            }
         }
         $sourceFile->statementList =
             \array_merge($sourceFile->statementList, $this->parseList($sourceFile, ParseContext::SourceElements));
@@ -192,6 +197,10 @@ private function parseList($parentNode, int $listParseContext) {
                 $element = $parseListElementFn($parentNode);
                 if ($element instanceof Node) {
                     $element->parent = $parentNode;
+                    if ($listParseContext === ParseContext::SourceElements && $element instanceof InlineHtml && $element->echoExpression) {
+                        $nodeArray[] = $element->echoExpression;
+                        $element->echoExpression = null;
+                    }
                 }
                 $nodeArray[] = $element;
                 continue;
@@ -3170,7 +3179,19 @@ private function parseInlineHtml($parentNode) {
         $inlineHtml->parent = $parentNode;
         $inlineHtml->scriptSectionEndTag = $this->eatOptional1(TokenKind::ScriptSectionEndTag);
         $inlineHtml->text = $this->eatOptional1(TokenKind::InlineHtml);
-        $inlineHtml->scriptSectionStartTag = $this->eatOptional1(TokenKind::ScriptSectionStartTag);
+        $inlineHtml->scriptSectionStartTag = $this->eatOptional(TokenKind::ScriptSectionStartTag, TokenKind::ScriptSectionStartWithEchoTag);
+
+        // This is the easiest way to represent `<?= "expr", "other" `
+        if (($inlineHtml->scriptSectionStartTag->kind ?? null) === TokenKind::ScriptSectionStartWithEchoTag)  {
+            $expressionList = $this->parseExpressionList($inlineHtml);
+            if ($expressionList) {
+                $echoExpression = new EchoExpression();
+                $echoExpression->expressions = $expressionList;
+                $echoExpression->parent = $inlineHtml;
+                // Deliberately leave echoKeyword as null instead of MissingToken
+                $inlineHtml->echoExpression = $echoExpression;
+           }
+        }
 
         return $inlineHtml;
     }

diff --git a/src/PhpTokenizer.php b/src/PhpTokenizer.php
@@ -83,6 +83,7 @@ public static function getTokensArrayFromContent(
             }
 
             switch ($tokenKind) {
+                // TODO: Remove?
                 case T_OPEN_TAG:
                     $arr[] = new Token(TokenKind::ScriptSectionStartTag, $fullStart, $start, $pos-$fullStart);
                     $start = $fullStart = $pos;
@@ -265,7 +266,7 @@ public static function getTokensArrayFromContent(
         T_DNUMBER => TokenKind::FloatingLiteralToken,
 
         T_OPEN_TAG => TokenKind::ScriptSectionStartTag,
-        T_OPEN_TAG_WITH_ECHO => TokenKind::ScriptSectionStartTag,
+        T_OPEN_TAG_WITH_ECHO => TokenKind::ScriptSectionStartWithEchoTag,
         T_CLOSE_TAG => TokenKind::ScriptSectionEndTag,
 
         T_INLINE_HTML => TokenKind::InlineHtml,

diff --git a/src/TokenKind.php b/src/TokenKind.php
@@ -167,7 +167,7 @@ class TokenKind {
 
     const ScriptSectionStartTag = 323;
     const ScriptSectionEndTag = 324;
-
+    const ScriptSectionStartWithEchoTag = 419;
 
     // TODO how to handle incremental parsing w/ this?
     const ScriptSectionPrependedText = 325;

diff --git a/src/TokenStringMaps.php b/src/TokenStringMaps.php
@@ -164,7 +164,7 @@ class TokenStringMaps {
         "<>" => TokenKind::LessThanGreaterThanToken,
         "..." => TokenKind::DotDotDotToken,
         "\\" => TokenKind::BackslashToken,
-        "<?=" => TokenKind::ScriptSectionStartTag, // TODO, technically not an operator
+        "<?=" => TokenKind::ScriptSectionStartWithEchoTag, // TODO, technically not an operator
         "<?php " => TokenKind::ScriptSectionStartTag, // TODO, technically not an operator
         "<?php\t" => TokenKind::ScriptSectionStartTag, // TODO add tests
         "<?php\n" => TokenKind::ScriptSectionStartTag,

diff --git a/tests/cases/parser/programStructure22.php.tree b/tests/cases/parser/programStructure22.php.tree
@@ -6,7 +6,7 @@
                     "scriptSectionEndTag": null,
                     "text": null,
                     "scriptSectionStartTag": {
-                        "kind": "ScriptSectionStartTag",
+                        "kind": "ScriptSectionStartWithEchoTag",
                         "textLength": 3
                     }
                 }

diff --git a/tests/cases/parser/programStructure5.php.tree b/tests/cases/parser/programStructure5.php.tree
@@ -6,7 +6,7 @@
                     "scriptSectionEndTag": null,
                     "text": null,
                     "scriptSectionStartTag": {
-                        "kind": "ScriptSectionStartTag",
+                        "kind": "ScriptSectionStartWithEchoTag",
                         "textLength": 3
                     }
                 }

diff --git a/tests/cases/parser/programStructure6.php.tree b/tests/cases/parser/programStructure6.php.tree
@@ -9,7 +9,7 @@
                         "textLength": 4
                     },
                     "scriptSectionStartTag": {
-                        "kind": "ScriptSectionStartTag",
+                        "kind": "ScriptSectionStartWithEchoTag",
                         "textLength": 3
                     }
                 }