Review all failing tests, add some test fixes, add some fixes

Author: dmsnell

src/wp-includes/html-api/class-wp-html-tag-processor.php

@@ -1268,6 +1268,8 @@ private function skip_script_data() {
 				}
 
 				if ( $this->bytes_already_parsed >= $doc_length ) {
+					$this->parser_state = self::STATE_INCOMPLETE;
+
 					return false;
 				}
 
@@ -1316,7 +1318,7 @@ private function parse_next_tag() {
 
 			$this->token_starts_at = $at;
 
-			if ( $at + 1 < $doc_length && '/' === $this->html[ $at + 1 ] ) {
+			if ( $doc_length > $at + 1 && '/' === $this->html[ $at + 1 ] ) {
 				$this->is_closing_tag = true;
 				++$at;
 			} else {
@@ -1366,7 +1368,7 @@ private function parse_next_tag() {
 				 * https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state
 				 */
 				if (
-					strlen( $html ) > $at + 3 &&
+					$doc_length > $at + 3 &&
 					'-' === $html[ $at + 2 ] &&
 					'-' === $html[ $at + 3 ]
 				) {
@@ -1516,7 +1518,7 @@ private function parse_next_tag() {
 			 * See https://html.spec.whatwg.org/#parse-error-invalid-first-character-of-tag-name
 			 */
 			if ( $this->is_closing_tag ) {
-				// No chance of finding a closer.
+				// No chance of finding a closer
 				if ( $at + 3 > $doc_length ) {
 					return false;
 				}

tests/phpunit/tests/html-api/wpHtmlProcessorHtml5lib.php

@@ -20,10 +20,13 @@ class Tests_HtmlApi_WpHtmlProcessorHtml5lib extends WP_UnitTestCase {
 		'adoption01/case4 - line 46'   => 'Unimplemented: Reconstruction of active formatting elements.',
 		'tests15/case1 - line 1'       => 'Unimplemented: Reconstruction of active formatting elements.',
 		'tests15/case2 - line 22'      => 'Unimplemented: Reconstruction of active formatting elements.',
+		'tests20/case38 - line 483'    => 'XMP is a special token an needs a closer.',
+		'tests20/case39 - line 497'    => "Closing P tag implicitly creates opener, which we don't visit.",
 		'tests23/case1 - line 1'       => 'Unimplemented: Reconstruction of active formatting elements.',
 		'tests23/case2 - line 41'      => 'Unimplemented: Reconstruction of active formatting elements.',
 		'tests23/case3 - line 69'      => 'Unimplemented: Reconstruction of active formatting elements.',
 		'tests23/case4 - line 101'     => 'Unimplemented: Reconstruction of active formatting elements.',
+		'tests26/case10 - line 263'    => 'Newline at end of tag triggers reconstruction of active formatting elements.',
 	);
 
 	/**
@@ -111,7 +114,10 @@ public static function build_html5_treelike_string( $fragment_context, $html ) {
 			$output .= "<{$t}>\n";
 		}
 
-		if ( WP_HTML_Processor::ERROR_UNSUPPORTED === $p->get_last_error() ) {
+		if (
+			$p->paused_at_incomplete_token() ||
+			WP_HTML_Processor::ERROR_UNSUPPORTED === $p->get_last_error()
+		) {
 			return null;
 		}