diff --git a/lib/compat/wordpress-6.5/html-api/class-gutenberg-html-tag-processor-6-5.php b/lib/compat/wordpress-6.5/html-api/class-gutenberg-html-tag-processor-6-5.php
index f14bc15adf999..de3823d2b2703 100644
--- a/lib/compat/wordpress-6.5/html-api/class-gutenberg-html-tag-processor-6-5.php
+++ b/lib/compat/wordpress-6.5/html-api/class-gutenberg-html-tag-processor-6-5.php
@@ -15,9 +15,6 @@
* - Prune the whitespace when removing classes/attributes: e.g. "a b c" -> "c" not " c".
* This would increase the size of the changes for some operations but leave more
* natural-looking output HTML.
- * - Decode HTML character references within class names when matching. E.g. match having
- * class `1<"2` needs to recognize `class="1<"2"`. Currently the Tag Processor
- * will fail to find the right tag if the class name is encoded as such.
* - Properly decode HTML character references in `get_attribute()`. PHP's
* `html_entity_decode()` is wrong in a couple ways: it doesn't account for the
* no-ambiguous-ampersand rule, and it improperly handles the way semicolons may
@@ -107,6 +104,56 @@
* given, it will return `true` (the only way to set `false` for an
* attribute is to remove it).
*
+ * #### When matching fails
+ *
+ * When `next_tag()` returns `false` it could mean different things:
+ *
+ * - The requested tag wasn't found in the input document.
+ * - The input document ended in the middle of an HTML syntax element.
+ *
+ * When a document ends in the middle of a syntax element it will pause
+ * the processor. This is to make it possible in the future to extend the
+ * input document and proceed - an important requirement for chunked
+ * streaming parsing of a document.
+ *
+ * Example:
+ *
+ * $processor = new WP_HTML_Tag_Processor( 'This
` inside an HTML comment.
+ * - STYLE content is raw text.
+ * - TITLE content is plain text but character references are decoded.
+ * - TEXTAREA content is plain text but character references are decoded.
+ * - XMP (deprecated) content is raw text.
+ *
* ### Modifying HTML attributes for a found tag
*
* Once you've found the start of an opening tag you can modify
@@ -241,9 +288,39 @@
* double-quoted strings, meaning that attributes on input with single-quoted or
* unquoted values will appear in the output with double-quotes.
*
+ * ### Scripting Flag
+ *
+ * The Tag Processor parses HTML with the "scripting flag" disabled. This means
+ * that it doesn't run any scripts while parsing the page. In a browser with
+ * JavaScript enabled, for example, the script can change the parse of the
+ * document as it loads. On the server, however, evaluating JavaScript is not
+ * only impractical, but also unwanted.
+ *
+ * Practically this means that the Tag Processor will descend into NOSCRIPT
+ * elements and process its child tags. Were the scripting flag enabled, such
+ * as in a typical browser, the contents of NOSCRIPT are skipped entirely.
+ *
+ * This allows the HTML API to process the content that will be presented in
+ * a browser when scripting is disabled, but it offers a different view of a
+ * page than most browser sessions will experience. E.g. the tags inside the
+ * NOSCRIPT disappear.
+ *
+ * ### Text Encoding
+ *
+ * The Tag Processor assumes that the input HTML document is encoded with a
+ * text encoding compatible with 7-bit ASCII's '<', '>', '&', ';', '/', '=',
+ * "'", '"', 'a' - 'z', 'A' - 'Z', and the whitespace characters ' ', tab,
+ * carriage-return, newline, and form-feed.
+ *
+ * In practice, this includes almost every single-byte encoding as well as
+ * UTF-8. Notably, however, it does not include UTF-16. If providing input
+ * that's incompatible, then convert the encoding beforehand.
+ *
* @since 6.2.0
* @since 6.2.1 Fix: Support for various invalid comments; attribute updates are case-insensitive.
* @since 6.3.2 Fix: Skip HTML-like content inside rawtext elements such as STYLE.
+ * @since 6.5.0 Pauses processor when input ends in an incomplete syntax token.
+ * Introduces "special" elements which act like void elements, e.g. STYLE.
*/
class Gutenberg_HTML_Tag_Processor_6_5 {
/**
@@ -316,6 +393,27 @@ class Gutenberg_HTML_Tag_Processor_6_5 {
*/
private $stop_on_tag_closers;
+ /**
+ * Specifies mode of operation of the parser at any given time.
+ *
+ * | State | Meaning |
+ * | --------------|----------------------------------------------------------------------|
+ * | *Ready* | The parser is ready to run. |
+ * | *Complete* | There is nothing left to parse. |
+ * | *Incomplete* | The HTML ended in the middle of a token; nothing more can be parsed. |
+ * | *Matched tag* | Found an HTML tag; it's possible to modify its attributes. |
+ *
+ * @since 6.5.0
+ *
+ * @see WP_HTML_Tag_Processor::STATE_READY
+ * @see WP_HTML_Tag_Processor::STATE_COMPLETE
+ * @see WP_HTML_Tag_Processor::STATE_INCOMPLETE
+ * @see WP_HTML_Tag_Processor::STATE_MATCHED_TAG
+ *
+ * @var string
+ */
+ private $parser_state = self::STATE_READY;
+
/**
* How many bytes from the original HTML document have been read and parsed.
*
@@ -544,6 +642,7 @@ public function __construct( $html ) {
* Finds the next tag matching the $query.
*
* @since 6.2.0
+ * @since 6.5.0 No longer processes incomplete tokens at end of document; pauses the processor at start of token.
*
* @param array|string|null $query {
* Optional. Which tag name to find, having which class, etc. Default is to find any tag.
@@ -562,90 +661,177 @@ public function next_tag( $query = null ) {
$already_found = 0;
do {
- if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
- return false;
- }
-
- // Find the next tag if it exists.
- if ( false === $this->parse_next_tag() ) {
- $this->bytes_already_parsed = strlen( $this->html );
-
+ if ( false === $this->next_token() ) {
return false;
}
- // Parse all of its attributes.
- while ( $this->parse_next_attribute() ) {
+ if ( self::STATE_MATCHED_TAG !== $this->parser_state ) {
continue;
}
- // Ensure that the tag closes before the end of the document.
- if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
- return false;
+ if ( $this->matches() ) {
+ ++$already_found;
}
+ } while ( $already_found < $this->sought_match_offset );
- $tag_ends_at = strpos( $this->html, '>', $this->bytes_already_parsed );
- if ( false === $tag_ends_at ) {
- return false;
- }
- $this->token_length = $tag_ends_at - $this->token_starts_at;
- $this->bytes_already_parsed = $tag_ends_at;
+ return true;
+ }
- // Finally, check if the parsed tag and its attributes match the search query.
- if ( $this->matches() ) {
- ++$already_found;
+ /**
+ * Finds the next token in the HTML document.
+ *
+ * An HTML document can be viewed as a stream of tokens,
+ * where tokens are things like HTML tags, HTML comments,
+ * text nodes, etc. This method finds the next token in
+ * the HTML document and returns whether it found one.
+ *
+ * If it starts parsing a token and reaches the end of the
+ * document then it will seek to the start of the last
+ * token and pause, returning `false` to indicate that it
+ * failed to find a complete token.
+ *
+ * Possible token types, based on the HTML specification:
+ *
+ * - an HTML tag, whether opening, closing, or void.
+ * - a text node - the plaintext inside tags.
+ * - an HTML comment.
+ * - a DOCTYPE declaration.
+ * - a processing instruction, e.g. ``.
+ *
+ * The Tag Processor currently only supports the tag token.
+ *
+ * @since 6.5.0
+ *
+ * @return bool Whether a token was parsed.
+ */
+ public function next_token() {
+ $this->get_updated_html();
+ $was_at = $this->bytes_already_parsed;
+
+ // Don't proceed if there's nothing more to scan.
+ if (
+ self::STATE_COMPLETE === $this->parser_state ||
+ self::STATE_INCOMPLETE === $this->parser_state
+ ) {
+ return false;
+ }
+
+ /*
+ * The next step in the parsing loop determines the parsing state;
+ * clear it so that state doesn't linger from the previous step.
+ */
+ $this->parser_state = self::STATE_READY;
+
+ if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
+ $this->parser_state = self::STATE_COMPLETE;
+ return false;
+ }
+
+ // Find the next tag if it exists.
+ if ( false === $this->parse_next_tag() ) {
+ if ( self::STATE_INCOMPLETE === $this->parser_state ) {
+ $this->bytes_already_parsed = $was_at;
}
- /*
- * For non-DATA sections which might contain text that looks like HTML tags but
- * isn't, scan with the appropriate alternative mode. Looking at the first letter
- * of the tag name as a pre-check avoids a string allocation when it's not needed.
- */
- $t = $this->html[ $this->tag_name_starts_at ];
- if (
- ! $this->is_closing_tag &&
+ return false;
+ }
+
+ // Parse all of its attributes.
+ while ( $this->parse_next_attribute() ) {
+ continue;
+ }
+
+ // Ensure that the tag closes before the end of the document.
+ if (
+ self::STATE_INCOMPLETE === $this->parser_state ||
+ $this->bytes_already_parsed >= strlen( $this->html )
+ ) {
+ // Does this appropriately clear state (parsed attributes)?
+ $this->parser_state = self::STATE_INCOMPLETE;
+ $this->bytes_already_parsed = $was_at;
+
+ return false;
+ }
+
+ $tag_ends_at = strpos( $this->html, '>', $this->bytes_already_parsed );
+ if ( false === $tag_ends_at ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+ $this->bytes_already_parsed = $was_at;
+
+ return false;
+ }
+ $this->parser_state = self::STATE_MATCHED_TAG;
+ $this->token_length = $tag_ends_at - $this->token_starts_at;
+ $this->bytes_already_parsed = $tag_ends_at;
+
+ /*
+ * For non-DATA sections which might contain text that looks like HTML tags but
+ * isn't, scan with the appropriate alternative mode. Looking at the first letter
+ * of the tag name as a pre-check avoids a string allocation when it's not needed.
+ */
+ $t = $this->html[ $this->tag_name_starts_at ];
+ if (
+ ! $this->is_closing_tag &&
+ (
+ 'i' === $t || 'I' === $t ||
+ 'n' === $t || 'N' === $t ||
+ 's' === $t || 'S' === $t ||
+ 't' === $t || 'T' === $t ||
+ 'x' === $t || 'X' === $t
+ )
+ ) {
+ $tag_name = $this->get_tag();
+
+ if ( 'SCRIPT' === $tag_name && ! $this->skip_script_data() ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+ $this->bytes_already_parsed = $was_at;
+
+ return false;
+ } elseif (
+ ( 'TEXTAREA' === $tag_name || 'TITLE' === $tag_name ) &&
+ ! $this->skip_rcdata( $tag_name )
+ ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+ $this->bytes_already_parsed = $was_at;
+
+ return false;
+ } elseif (
(
- 'i' === $t || 'I' === $t ||
- 'n' === $t || 'N' === $t ||
- 's' === $t || 'S' === $t ||
- 't' === $t || 'T' === $t
- ) ) {
- $tag_name = $this->get_tag();
-
- if ( 'SCRIPT' === $tag_name && ! $this->skip_script_data() ) {
- $this->bytes_already_parsed = strlen( $this->html );
- return false;
- } elseif (
- ( 'TEXTAREA' === $tag_name || 'TITLE' === $tag_name ) &&
- ! $this->skip_rcdata( $tag_name )
- ) {
- $this->bytes_already_parsed = strlen( $this->html );
- return false;
- } elseif (
- (
- 'IFRAME' === $tag_name ||
- 'NOEMBED' === $tag_name ||
- 'NOFRAMES' === $tag_name ||
- 'NOSCRIPT' === $tag_name ||
- 'STYLE' === $tag_name
- ) &&
- ! $this->skip_rawtext( $tag_name )
- ) {
- /*
- * "XMP" should be here too but its rules are more complicated and require the
- * complexity of the HTML Processor (it needs to close out any open P element,
- * meaning it can't be skipped here or else the HTML Processor will lose its
- * place). For now, it can be ignored as it's a rare HTML tag in practice and
- * any normative HTML should be using PRE instead.
- */
- $this->bytes_already_parsed = strlen( $this->html );
- return false;
- }
+ 'IFRAME' === $tag_name ||
+ 'NOEMBED' === $tag_name ||
+ 'NOFRAMES' === $tag_name ||
+ 'STYLE' === $tag_name ||
+ 'XMP' === $tag_name
+ ) &&
+ ! $this->skip_rawtext( $tag_name )
+ ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+ $this->bytes_already_parsed = $was_at;
+
+ return false;
}
- } while ( $already_found < $this->sought_match_offset );
+ }
return true;
}
+ /**
+ * Whether the processor paused because the input HTML document ended
+ * in the middle of a syntax element, such as in the middle of a tag.
+ *
+ * Example:
+ *
+ * $processor = new WP_HTML_Tag_Processor( '' === $html[ $at + 1 ] ) {
@@ -1276,6 +1502,8 @@ private function parse_next_tag() {
if ( '?' === $html[ $at + 1 ] ) {
$closer_at = strpos( $html, '>', $at + 2 );
if ( false === $closer_at ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1290,8 +1518,15 @@ private function parse_next_tag() {
* See https://html.spec.whatwg.org/#parse-error-invalid-first-character-of-tag-name
*/
if ( $this->is_closing_tag ) {
+ // No chance of finding a closer.
+ if ( $at + 3 > $doc_length ) {
+ return false;
+ }
+
$closer_at = strpos( $html, '>', $at + 3 );
if ( false === $closer_at ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1316,6 +1551,8 @@ private function parse_next_attribute() {
// Skip whitespace and slashes.
$this->bytes_already_parsed += strspn( $this->html, " \t\f\r\n/", $this->bytes_already_parsed );
if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1338,11 +1575,15 @@ private function parse_next_attribute() {
$attribute_name = substr( $this->html, $attribute_start, $name_length );
$this->bytes_already_parsed += $name_length;
if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
$this->skip_whitespace();
if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1351,6 +1592,8 @@ private function parse_next_attribute() {
++$this->bytes_already_parsed;
$this->skip_whitespace();
if ( $this->bytes_already_parsed >= strlen( $this->html ) ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1377,6 +1620,8 @@ private function parse_next_attribute() {
}
if ( $attribute_end >= strlen( $this->html ) ) {
+ $this->parser_state = self::STATE_INCOMPLETE;
+
return false;
}
@@ -1443,7 +1688,6 @@ private function skip_whitespace() {
* @since 6.2.0
*/
private function after_tag() {
- $this->get_updated_html();
$this->token_starts_at = null;
$this->token_length = null;
$this->tag_name_starts_at = null;
@@ -1786,6 +2030,10 @@ private static function sort_start_ascending( $a, $b ) {
* @return string|boolean|null Value of enqueued update if present, otherwise false.
*/
private function get_enqueued_attribute_value( $comparable_name ) {
+ if ( self::STATE_MATCHED_TAG !== $this->parser_state ) {
+ return false;
+ }
+
if ( ! isset( $this->lexical_updates[ $comparable_name ] ) ) {
return false;
}
@@ -1853,7 +2101,7 @@ private function get_enqueued_attribute_value( $comparable_name ) {
* @return string|true|null Value of attribute or `null` if not available. Boolean attributes return `true`.
*/
public function get_attribute( $name ) {
- if ( null === $this->tag_name_starts_at ) {
+ if ( self::STATE_MATCHED_TAG !== $this->parser_state ) {
return null;
}
@@ -1933,7 +2181,10 @@ public function get_attribute( $name ) {
* @return array|null List of attribute names, or `null` when no tag opener is matched.
*/
public function get_attribute_names_with_prefix( $prefix ) {
- if ( $this->is_closing_tag || null === $this->tag_name_starts_at ) {
+ if (
+ self::STATE_MATCHED_TAG !== $this->parser_state ||
+ $this->is_closing_tag
+ ) {
return null;
}
@@ -1965,7 +2216,7 @@ public function get_attribute_names_with_prefix( $prefix ) {
* @return string|null Name of currently matched tag in input HTML, or `null` if none found.
*/
public function get_tag() {
- if ( null === $this->tag_name_starts_at ) {
+ if ( self::STATE_MATCHED_TAG !== $this->parser_state ) {
return null;
}
@@ -1992,7 +2243,7 @@ public function get_tag() {
* @return bool Whether the currently matched tag contains the self-closing flag.
*/
public function has_self_closing_flag() {
- if ( ! $this->tag_name_starts_at ) {
+ if ( self::STATE_MATCHED_TAG !== $this->parser_state ) {
return false;
}
@@ -2024,7 +2275,10 @@ public function has_self_closing_flag() {
* @return bool Whether the current tag is a tag closer.
*/
public function is_tag_closer() {
- return $this->is_closing_tag;
+ return (
+ self::STATE_MATCHED_TAG === $this->parser_state &&
+ $this->is_closing_tag
+ );
}
/**
@@ -2044,7 +2298,10 @@ public function is_tag_closer() {
* @return bool Whether an attribute value was set.
*/
public function set_attribute( $name, $value ) {
- if ( $this->is_closing_tag || null === $this->tag_name_starts_at ) {
+ if (
+ self::STATE_MATCHED_TAG !== $this->parser_state ||
+ $this->is_closing_tag
+ ) {
return false;
}
@@ -2177,7 +2434,10 @@ public function set_attribute( $name, $value ) {
* @return bool Whether an attribute was removed.
*/
public function remove_attribute( $name ) {
- if ( $this->is_closing_tag ) {
+ if (
+ self::STATE_MATCHED_TAG !== $this->parser_state ||
+ $this->is_closing_tag
+ ) {
return false;
}
@@ -2254,13 +2514,14 @@ public function remove_attribute( $name ) {
* @return bool Whether the class was set to be added.
*/
public function add_class( $class_name ) {
- if ( $this->is_closing_tag ) {
+ if (
+ self::STATE_MATCHED_TAG !== $this->parser_state ||
+ $this->is_closing_tag
+ ) {
return false;
}
- if ( null !== $this->tag_name_starts_at ) {
- $this->classname_updates[ $class_name ] = self::ADD_CLASS;
- }
+ $this->classname_updates[ $class_name ] = self::ADD_CLASS;
return true;
}
@@ -2274,7 +2535,10 @@ public function add_class( $class_name ) {
* @return bool Whether the class was set to be removed.
*/
public function remove_class( $class_name ) {
- if ( $this->is_closing_tag ) {
+ if (
+ self::STATE_MATCHED_TAG !== $this->parser_state ||
+ $this->is_closing_tag
+ ) {
return false;
}
@@ -2480,4 +2744,57 @@ private function matches() {
return true;
}
+
+ /**
+ * Parser Ready State
+ *
+ * Indicates that the parser is ready to run and waiting for a state transition.
+ * It may not have started yet, or it may have just finished parsing a token and
+ * is ready to find the next one.
+ *
+ * @since 6.5.0
+ *
+ * @access private
+ */
+ const STATE_READY = 'STATE_READY';
+
+ /**
+ * Parser Complete State
+ *
+ * Indicates that the parser has reached the end of the document and there is
+ * nothing left to scan. It finished parsing the last token completely.
+ *
+ * @since 6.5.0
+ *
+ * @access private
+ */
+ const STATE_COMPLETE = 'STATE_COMPLETE';
+
+ /**
+ * Parser Incomplete State
+ *
+ * Indicates that the parser has reached the end of the document before finishing
+ * a token. It started parsing a token but there is a possibility that the input
+ * HTML document was truncated in the middle of a token.
+ *
+ * The parser is reset at the start of the incomplete token and has paused. There
+ * is nothing more than can be scanned unless provided a more complete document.
+ *
+ * @since 6.5.0
+ *
+ * @access private
+ */
+ const STATE_INCOMPLETE = 'STATE_INCOMPLETE';
+
+ /**
+ * Parser Matched Tag State
+ *
+ * Indicates that the parser has found an HTML tag and it's possible to get
+ * the tag name and read or modify its attributes (if it's not a closing tag).
+ *
+ * @since 6.5.0
+ *
+ * @access private
+ */
+ const STATE_MATCHED_TAG = 'STATE_MATCHED_TAG';
}
diff --git a/phpcs.xml.dist b/phpcs.xml.dist
index 21f3fcb8baee1..1eee805dc8087 100644
--- a/phpcs.xml.dist
+++ b/phpcs.xml.dist
@@ -58,6 +58,11 @@
./vendor/*./test/php/gutenberg-coding-standards/*
+
+ ./lib/compat/wordpress-*/html-api/*.php
+
/phpunit/*