Fixed bug #1718 : Unclosed strings at EOF sometimes tokenized as T_WH…

…ITESPACE by the JS tokenizer
squizlabs · Nov 8, 2017 · 5ad186a · 5ad186a
1 parent b58baf0
commit 5ad186a
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 11 deletions.
diff --git a/package.xml b/package.xml
@@ -75,6 +75,7 @@ http://pear.php.net/dtd/package-2.0.xsd">
     -- Override the 'requiredSpacesBeforeColon' setting in a ruleset.xml file to change
     -- Default remains at 1
     -- Thanks to Nikola Kovacs for the patch
+  - Fixed bug #1718 : Unclosed strings at EOF sometimes tokenized as T_WHITESPACE by the JS tokenizer
   - Fixed bug #1731 : Directory exclusions do not work as expected when a single file name is passed to phpcs
  </notes>
  <contents>

diff --git a/src/Tokenizers/JS.php b/src/Tokenizers/JS.php
@@ -709,18 +709,33 @@ public function tokenize($string)
         }//end for
 
         if (empty($buffer) === false) {
-            // Buffer contains whitespace from the end of the file.
-            $tokens[] = array(
-                         'code'    => T_WHITESPACE,
-                         'type'    => 'T_WHITESPACE',
-                         'content' => str_replace("\n", $this->eolChar, $buffer),
-                        );
+            if ($inString !== '') {
+                // The string did not end before the end of the file,
+                // which means there was probably a syntax error somewhere.
+                $tokens[] = array(
+                             'code'    => T_STRING,
+                             'type'    => 'T_STRING',
+                             'content' => str_replace("\n", $this->eolChar, $buffer),
+                            );
 
-            if (PHP_CODESNIFFER_VERBOSITY > 1) {
-                $content = Util\Common::prepareForOutput($buffer);
-                echo "\t=> Added token T_WHITESPACE ($content)".PHP_EOL;
-            }
-        }
+                if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                    $content = Util\Common::prepareForOutput($buffer);
+                    echo "\t=> Added token T_STRING ($content)".PHP_EOL;
+                }
+            } else {
+                // Buffer contains whitespace from the end of the file.
+                $tokens[] = array(
+                             'code'    => T_WHITESPACE,
+                             'type'    => 'T_WHITESPACE',
+                             'content' => str_replace("\n", $this->eolChar, $buffer),
+                            );
+
+                if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                    $content = Util\Common::prepareForOutput($buffer);
+                    echo "\t=> Added token T_WHITESPACE ($content)".PHP_EOL;
+                }
+            }//end if
+        }//end if
 
         $tokens[] = array(
                      'code'    => T_CLOSE_TAG,