Preserve whitespace formatting of text nodes

Author: JanTvrdik

src/Lexer/Lexer.php

@@ -139,10 +139,10 @@ private function initialize(): void
 			self::TOKEN_EQUAL => '=',
 			self::TOKEN_COLON => ':',
 
-			self::TOKEN_OPEN_PHPDOC => '/\\*\\*(?=\\s)',
+			self::TOKEN_OPEN_PHPDOC => '/\\*\\*(?=\\s)\\x20?+',
 			self::TOKEN_CLOSE_PHPDOC => '\\*/',
 			self::TOKEN_PHPDOC_TAG => '@[a-z][a-z0-9-]*+',
-			self::TOKEN_PHPDOC_EOL => '\\r?+\\n[\\x09\\x20]*+(?:\\*(?!/))?',
+			self::TOKEN_PHPDOC_EOL => '\\r?+\\n[\\x09\\x20]*+(?:\\*(?!/)\\x20?+)?',
 
 			self::TOKEN_FLOAT => '(?:-?[0-9]++\\.[0-9]*+(?:e-?[0-9]++)?)|(?:-?[0-9]*+\\.[0-9]++(?:e-?[0-9]++)?)|(?:-?[0-9]++e-?[0-9]++)',
 			self::TOKEN_INTEGER => '-?(?:(?:0b[0-1]++)|(?:0o[0-7]++)|(?:0x[0-9a-f]++)|(?:[0-9]++))',

src/Parser/PhpDocParser.php

@@ -61,39 +61,27 @@ private function parseChild(TokenIterator $tokens): Ast\PhpDoc\PhpDocChildNode
 	private function parseText(TokenIterator $tokens): Ast\PhpDoc\PhpDocTextNode
 	{
 		$text = '';
-		while (true) {
-			// If we received a Lexer::TOKEN_PHPDOC_EOL, exit early to prevent
-			// them from being processed.
-			$currentTokenType = $tokens->currentTokenType();
-			if ($currentTokenType === Lexer::TOKEN_PHPDOC_EOL) {
-				break;
-			}
-			$text .= $tokens->joinUntil(Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END);
-			$text = rtrim($text, " \t");
 
-			// If we joined until TOKEN_PHPDOC_EOL, peak at the next tokens to see
-			// if we have a multiline string to join.
-			$currentTokenType = $tokens->currentTokenType();
-			if ($currentTokenType !== Lexer::TOKEN_PHPDOC_EOL) {
+		while (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) {
+			$text .= $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END);
+
+			if (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) {
 				break;
 			}
 
-			// Peek at the next token to determine if it is more text that needs
-			// to be combined.
 			$tokens->pushSavePoint();
 			$tokens->next();
-			$currentTokenType = $tokens->currentTokenType();
-			if ($currentTokenType !== Lexer::TOKEN_IDENTIFIER) {
+
+			if ($tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_TAG) || $tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL) || $tokens->isCurrentTokenType(Lexer::TOKEN_CLOSE_PHPDOC) || $tokens->isCurrentTokenType(Lexer::TOKEN_END)) {
 				$tokens->rollback();
 				break;
 			}
 
-			// There's more text on a new line, ensure spacing.
+			$tokens->dropSavePoint();
 			$text .= "\n";
 		}
-		$text = trim($text, " \t");
 
-		return new Ast\PhpDoc\PhpDocTextNode($text);
+		return new Ast\PhpDoc\PhpDocTextNode(trim($text, " \t"));
 	}
 
 

tests/PHPStan/Parser/PhpDocParserTest.php

@@ -1242,9 +1242,9 @@ public function provideDeprecatedTagsData(): \Iterator
 				new PhpDocTagNode(
 					'@deprecated',
 					new DeprecatedTagValueNode('in Drupal 8.6.0 and will be removed before Drupal 9.0.0. In
-Drupal 9 there will be no way to set the status and in Drupal 8 this
-ability has been removed because mb_*() functions are supplied using
-Symfony\'s polyfill.')
+  Drupal 9 there will be no way to set the status and in Drupal 8 this
+  ability has been removed because mb_*() functions are supplied using
+  Symfony\'s polyfill.')
 				),
 			]),
 		];
@@ -1271,9 +1271,9 @@ public function provideDeprecatedTagsData(): \Iterator
 				new PhpDocTagNode(
 					'@deprecated',
 					new DeprecatedTagValueNode('in Drupal 8.6.0 and will be removed before Drupal 9.0.0. In
-Drupal 9 there will be no way to set the status and in Drupal 8 this
-ability has been removed because mb_*() functions are supplied using
-Symfony\'s polyfill.')
+  Drupal 9 there will be no way to set the status and in Drupal 8 this
+  ability has been removed because mb_*() functions are supplied using
+  Symfony\'s polyfill.')
 				),
 			]),
 		];
@@ -3122,29 +3122,29 @@ public function provideRealWorldExampleData(): \Iterator
 						''
 					)
 				),
-				new PhpDocTextNode('An empty array if there is no schema, or an associative array with the
-following key/value pairs:'),
-				new PhpDocTextNode('- columns: An array of Schema API column specifications, keyed by column
-name. The columns need to be a subset of the properties defined in
-propertyDefinitions(). The \'not null\' property is ignored if present,
-as it is determined automatically by the storage controller depending
-on the table layout and the property definitions. It is recommended to
-avoid having the column definitions depend on field settings when
-possible. No assumptions should be made on how storage engines
-internally use the original column name to structure their storage.'),
-				new PhpDocTextNode('- unique keys: (optional) An array of Schema API unique key definitions.
-Only columns that appear in the \'columns\' array are allowed.'),
-				new PhpDocTextNode('- indexes: (optional) An array of Schema API index definitions. Only
-columns that appear in the \'columns\' array are allowed. Those indexes
-will be used as default indexes. Field definitions can specify
-additional indexes or, at their own risk, modify the default indexes
-specified by the field-type module. Some storage engines might not
-support indexes.'),
-				new PhpDocTextNode('- foreign keys: (optional) An array of Schema API foreign key
-definitions. Note, however, that the field data is not necessarily
-stored in SQL. Also, the possible usage is limited, as you cannot
-specify another field as related, only existing SQL tables,
-such as {taxonomy_term_data}.'),
+				new PhpDocTextNode("An empty array if there is no schema, or an associative array with the
+  following key/value pairs:
+  - columns: An array of Schema API column specifications, keyed by column
+    name. The columns need to be a subset of the properties defined in
+    propertyDefinitions(). The 'not null' property is ignored if present,
+    as it is determined automatically by the storage controller depending
+    on the table layout and the property definitions. It is recommended to
+    avoid having the column definitions depend on field settings when
+    possible. No assumptions should be made on how storage engines
+    internally use the original column name to structure their storage.
+  - unique keys: (optional) An array of Schema API unique key definitions.
+    Only columns that appear in the 'columns' array are allowed.
+  - indexes: (optional) An array of Schema API index definitions. Only
+    columns that appear in the 'columns' array are allowed. Those indexes
+    will be used as default indexes. Field definitions can specify
+    additional indexes or, at their own risk, modify the default indexes
+    specified by the field-type module. Some storage engines might not
+    support indexes.
+  - foreign keys: (optional) An array of Schema API foreign key
+    definitions. Note, however, that the field data is not necessarily
+    stored in SQL. Also, the possible usage is limited, as you cannot
+    specify another field as related, only existing SQL tables,
+    such as {taxonomy_term_data}."),
 			]),
 		];
 
@@ -3172,14 +3172,14 @@ public function provideRealWorldExampleData(): \Iterator
 				new PhpDocTextNode('Parses a chunked request and return relevant information.'),
 				new PhpDocTextNode(''),
 				new PhpDocTextNode('This function must return an array containing the following
-keys and their corresponding values:'),
-				new PhpDocTextNode('- last: Wheter this is the last chunk of the uploaded file'),
-				new PhpDocTextNode('- uuid: A unique id which distinguishes two uploaded files
-This uuid must stay the same among the task of
-uploading a chunked file.'),
-				new PhpDocTextNode('- index: A numerical representation of the currently uploaded
-chunk. Must be higher that in the previous request.'),
-				new PhpDocTextNode('- orig: The original file name.'),
+ keys and their corresponding values:
+   - last: Wheter this is the last chunk of the uploaded file
+   - uuid: A unique id which distinguishes two uploaded files
+           This uuid must stay the same among the task of
+           uploading a chunked file.
+   - index: A numerical representation of the currently uploaded
+           chunk. Must be higher that in the previous request.
+   - orig: The original file name.'),
 				new PhpDocTextNode(''),
 				new PhpDocTagNode(
 					'@param',
@@ -3201,6 +3201,30 @@ public function provideRealWorldExampleData(): \Iterator
 			]),
 		];
 
+		yield [
+			'Description with indented <code>',
+			"/**
+			  * Finder allows searching through directory trees using iterator.
+			  *
+			  * <code>
+			  * Finder::findFiles('*.php')
+			  *     ->size('> 10kB')
+			  *     ->from('.')
+			  *     ->exclude('temp');
+			  * </code>
+			  */",
+			new PhpDocNode([
+				new PhpDocTextNode('Finder allows searching through directory trees using iterator.'),
+				new PhpDocTextNode(''),
+				new PhpDocTextNode("<code>
+Finder::findFiles('*.php')
+    ->size('> 10kB')
+    ->from('.')
+    ->exclude('temp');
+</code>"),
+			]),
+		];
+
 		yield [
 			'string literals in @return',
 			"/** @return 'foo'|'bar' */",