HTML API: Handle content after BODY, HTML where possible

phpstan.neon.dist

@@ -165,7 +165,7 @@ parameters:
 		-
 			message: '#^Unreachable statement \- code above always terminates\.$#'
 			identifier: deadCode.unreachable
-			count: 15
+			count: 12
 			path: src/wp-includes/html-api/class-wp-html-processor.php
 		-
 			message: '#^If condition is always true\.$#'

src/wp-includes/html-api/class-wp-html-processor.php

@@ -262,6 +262,12 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor {
 	 */
 	private $context_node = null;
 
+	/** @var bool */
+	private $has_produced_after_body_content = false;
+
+	/** @var bool */
+	private $has_produced_after_html_content = false;
+
 	/*
 	 * Public Interface Functions
 	 */
@@ -4366,6 +4372,9 @@ private function step_after_body(): bool {
 			 */
 			case '#text':
 				if ( parent::TEXT_IS_WHITESPACE === $this->text_node_classification ) {
+					if ( $this->has_produced_after_body_content || $this->has_produced_after_html_content ) {
+						$this->bail( 'Cannot produce out-of-order content.' );
+					}
 					return $this->step_in_body();
 				}
 				goto after_body_anything_else;
@@ -4378,8 +4387,15 @@ private function step_after_body(): bool {
 			case '#funky-comment':
 			case '#presumptuous-tag':
 			case '#processing-instruction':
-				$this->bail( 'Content outside of BODY is unsupported.' );
-				break;
+				if ( $this->has_produced_after_html_content ) {
+					$this->bail( 'Cannot produce out-of-order content.' );
+				}
+				if ( ! $this->has_produced_after_body_content ) {
+					$this->state->stack_of_open_elements->pop_until( 'BODY' );
+					$this->has_produced_after_body_content = true;
+				}
+				$this->insert_html_element( $this->state->current_token );
+				return true;
 
 			/*
 			 * > A DOCTYPE token
@@ -4422,6 +4438,9 @@ private function step_after_body(): bool {
 		 * > Parse error. Switch the insertion mode to "in body" and reprocess the token.
 		 */
 		after_body_anything_else:
+		if ( $this->has_produced_after_body_content || $this->has_produced_after_html_content ) {
+			$this->bail( 'Cannot return to in body when content has been produced outside of body.' );
+		}
 		$this->state->insertion_mode = WP_HTML_Processor_State::INSERTION_MODE_IN_BODY;
 		return $this->step( self::REPROCESS_CURRENT_NODE );
 	}
@@ -4460,6 +4479,9 @@ private function step_in_frameset(): bool {
 			 */
 			case '#text':
 				if ( parent::TEXT_IS_WHITESPACE === $this->text_node_classification ) {
+					if ( $this->has_produced_after_body_content || $this->has_produced_after_html_content ) {
+						$this->bail( 'Cannot produce out-of-order content.' );
+					}
 					return $this->step_in_body();
 				}
 				$this->bail( 'Non-whitespace characters cannot be handled in frameset.' );
@@ -4582,6 +4604,9 @@ private function step_after_frameset(): bool {
 			 */
 			case '#text':
 				if ( parent::TEXT_IS_WHITESPACE === $this->text_node_classification ) {
+					if ( $this->has_produced_after_body_content || $this->has_produced_after_html_content ) {
+						$this->bail( 'Cannot produce out-of-order content.' );
+					}
 					return $this->step_in_body();
 				}
 				$this->bail( 'Non-whitespace characters cannot be handled in after frameset' );
@@ -4667,8 +4692,14 @@ private function step_after_after_body(): bool {
 			case '#funky-comment':
 			case '#presumptuous-tag':
 			case '#processing-instruction':
-				$this->bail( 'Content outside of HTML is unsupported.' );
-				break;
+				if ( ! $this->has_produced_after_html_content ) {
+					while ( $this->state->stack_of_open_elements->pop() ) {
+						// Just pop while we can.
+					}
+					$this->has_produced_after_html_content = true;
+				}
+				$this->insert_html_element( $this->state->current_token );
+				return true;
 
 			/*
 			 * > A DOCTYPE token
@@ -4688,6 +4719,9 @@ private function step_after_after_body(): bool {
 			 */
 			case '#text':
 				if ( parent::TEXT_IS_WHITESPACE === $this->text_node_classification ) {
+					if ( $this->has_produced_after_body_content || $this->has_produced_after_html_content ) {
+						$this->bail( 'Cannot produce out-of-order content.' );
+					}
 					return $this->step_in_body();
 				}
 				goto after_after_body_anything_else;
@@ -4697,6 +4731,9 @@ private function step_after_after_body(): bool {
 		 * > Parse error. Switch the insertion mode to "in body" and reprocess the token.
 		 */
 		after_after_body_anything_else:
+		if ( $this->has_produced_after_body_content || $this->has_produced_after_html_content ) {
+			$this->bail( 'Cannot return to in body when content has been produced outside of body.' );
+		}
 		$this->state->insertion_mode = WP_HTML_Processor_State::INSERTION_MODE_IN_BODY;
 		return $this->step( self::REPROCESS_CURRENT_NODE );
 	}
@@ -4732,8 +4769,14 @@ private function step_after_after_frameset(): bool {
 			case '#funky-comment':
 			case '#presumptuous-tag':
 			case '#processing-instruction':
-				$this->bail( 'Content outside of HTML is unsupported.' );
-				break;
+				if ( ! $this->has_produced_after_html_content ) {
+					while ( $this->state->stack_of_open_elements->pop() ) {
+						// Just pop while we can.
+					}
+					$this->has_produced_after_html_content = true;
+				}
+				$this->insert_html_element( $this->state->current_token );
+				return true;
 
 			/*
 			 * > A DOCTYPE token
@@ -4756,6 +4799,9 @@ private function step_after_after_frameset(): bool {
 			 */
 			case '#text':
 				if ( parent::TEXT_IS_WHITESPACE === $this->text_node_classification ) {
+					if ( $this->has_produced_after_body_content || $this->has_produced_after_html_content ) {
+						$this->bail( 'Cannot produce out-of-order content.' );
+					}
 					return $this->step_in_body();
 				}
 				$this->bail( 'Non-whitespace characters cannot be handled in after after frameset.' );
@@ -4765,6 +4811,9 @@ private function step_after_after_frameset(): bool {
 			 * > A start tag whose tag name is "noframes"
 			 */
 			case '+NOFRAMES':
+				if ( $this->has_produced_after_body_content || $this->has_produced_after_html_content ) {
+					$this->bail( 'Cannot produce out-of-order content.' );
+				}
 				return $this->step_in_head();
 		}
 
@@ -5575,6 +5624,8 @@ public function seek( $bookmark_name ): bool {
 			$this->state->current_token                     = null;
 			$this->current_element                          = null;
 			$this->element_queue                            = array();
+			$this->has_produced_after_body_content          = false;
+			$this->has_produced_after_html_content          = false;
 
 			/*
 			 * The absence of a context node indicates a full parse.

tests/phpunit/tests/html-api/wpHtmlProcessor-bookmark.php

@@ -170,4 +170,35 @@ public function test_bookmarks_not_allowed_on_virtual_nodes() {
 		$this->assertTrue( $processor->next_token() );
 		$this->assertTrue( $processor->set_bookmark( 'mark' ) );
 	}
+
+	/**
+	 * Ensures that after a backward seek, content after BODY and HTML parses as it did the first time.
+	 *
+	 * @ticket 65967
+	 *
+	 * @covers ::seek
+	 */
+	public function test_seek_backward_reparses_content_after_body_and_html() {
+		$processor = WP_HTML_Processor::create_full_parser( '<html><body><p>x</p></body><!--a--></html><!--b-->' );
+		$this->assertTrue( $processor->next_tag( 'P' ) );
+		$this->assertTrue( $processor->set_bookmark( 'p' ), 'Failed to set bookmark "p".' );
+
+		$walk = static function () use ( $processor ) {
+			$tokens = array();
+			while ( $processor->next_token() ) {
+				$closer   = $processor->is_tag_closer() ? '/' : '';
+				$tokens[] = $closer . $processor->get_token_name() . ' ' . implode( ' > ', $processor->get_breadcrumbs() );
+			}
+			return $tokens;
+		};
+
+		$first_pass = $walk();
+		$this->assertNull( $processor->get_last_error(), 'Failed to parse the document on the first pass.' );
+
+		$this->assertTrue( $processor->seek( 'p' ), 'Failed to seek to bookmark "p".' );
+
+		$second_pass = $walk();
+		$this->assertSame( $first_pass, $second_pass, 'Found different tokens after seeking backward.' );
+		$this->assertNull( $processor->get_last_error(), 'Failed to parse the document after seeking backward.' );
+	}
 }

tests/phpunit/tests/html-api/wpHtmlProcessor-serialize.php

@@ -552,6 +552,105 @@ public static function data_provider_serialize_doctype() {
 		);
 	}
 
+	/**
+	 * Ensures that full documents serialize content after the BODY and HTML end tags
+	 * at its place in the document tree.
+	 *
+	 * @ticket 65967
+	 *
+	 * @dataProvider data_provider_serialize_content_after_body_and_html
+	 *
+	 * @covers ::serialize
+	 *
+	 * @param string $html     Full HTML document.
+	 * @param string $expected Expected serialization.
+	 */
+	public function test_full_document_serialize_includes_content_after_body_and_html( string $html, string $expected ) {
+		$processor = WP_HTML_Processor::create_full_parser( $html );
+		$this->assertSame( $expected, $processor->serialize() );
+	}
+
+	/**
+	 * Data provider.
+	 *
+	 * @return array[]
+	 */
+	public static function data_provider_serialize_content_after_body_and_html(): array {
+		return array(
+			'Comment after BODY'                 => array(
+				'<html><body></body><!--c-->',
+				'<html><head></head><body></body><!--c--></html>',
+			),
+			'Processing instruction after BODY'  => array(
+				'<html><body></body><?pi data?>',
+				'<html><head></head><body></body><?pi data?></html>',
+			),
+			'Whitespace and comment after BODY'  => array(
+				'<html><body></body> <!--c-->',
+				'<html><head></head><body> </body><!--c--></html>',
+			),
+			'Comment after unclosed P and BODY'  => array(
+				'<p>x</body><!--c-->',
+				'<html><head></head><body><p>x</p></body><!--c--></html>',
+			),
+			'Comment after HTML'                 => array(
+				'<html><body></body></html><!--c-->',
+				'<html><head></head><body></body></html><!--c-->',
+			),
+			'Processing instruction after HTML'  => array(
+				'<html><body></body></html><?pi data?>',
+				'<html><head></head><body></body></html><?pi data?>',
+			),
+			'Whitespace and comments after HTML' => array(
+				'<html><body></body> </html> <!--a--><!--b-->',
+				'<html><head></head><body>  </body></html><!--a--><!--b-->',
+			),
+			'Comments after BODY and after HTML' => array(
+				'<html><body></body><!--a--></html><!--b-->',
+				'<html><head></head><body></body><!--a--></html><!--b-->',
+			),
+			'Comment after FRAMESET and HTML'    => array(
+				'<frameset></frameset></html><!--c-->',
+				'<html><head></head><frameset></frameset></html><!--c-->',
+			),
+			'Document with a trailing comment'   => array(
+				'<!DOCTYPE html><html><head><title>t</title></head><body><p>x</p></body></html><!-- generated -->',
+				'<!DOCTYPE html><html><head><title>t</title></head><body><p>x</p></body></html><!-- generated -->',
+			),
+		);
+	}
+
+	/**
+	 * Ensures that normalize(), which parses its input as a BODY fragment, keeps
+	 * content after BODY and HTML end tags inside BODY.
+	 *
+	 * @ticket 65967
+	 *
+	 * @dataProvider data_provider_normalize_content_after_body_and_html
+	 *
+	 * @covers ::normalize
+	 *
+	 * @param string $html     HTML fragment.
+	 * @param string $expected Expected normalized HTML.
+	 */
+	public function test_normalize_keeps_content_after_body_and_html_in_body( string $html, string $expected ) {
+		$this->assertSame( $expected, WP_HTML_Processor::normalize( $html ) );
+	}
+
+	/**
+	 * Data provider.
+	 *
+	 * @return array[]
+	 */
+	public static function data_provider_normalize_content_after_body_and_html(): array {
+		return array(
+			'Comment after BODY'                  => array( '<p>x</p></body><!--c-->', '<p>x</p><!--c-->' ),
+			'Comment after unclosed P and BODY'   => array( '<p>x</body><!--c-->', '<p>x<!--c--></p>' ),
+			'Whitespace after comment after BODY' => array( '<p>x</p></body><!--c--> ', '<p>x</p><!--c--> ' ),
+			'Processing instruction and newline after HTML' => array( "<p>x</p></body></html><?pi data?>\n", "<p>x</p><?pi data?>\n" ),
+		);
+	}
+
 	/**
 	 * Ensures that leading newlines in PRE, LISTING, and TEXTAREA elements are normalized
 	 * according to their parsing namespace, and that normalization is idempotent in these cases.

tests/phpunit/tests/html-api/wpHtmlProcessor.php

@@ -1226,4 +1226,54 @@ public function test_prevents_unbounded_bookmarking() {
 			$processor->set_bookmark( 'beyond the limit' )
 		);
 	}
+
+	/**
+	 * Ensures that after visiting a node outside of BODY, the processor bails
+	 * on any later token it cannot visit in document order.
+	 *
+	 * @ticket 65967
+	 *
+	 * @dataProvider data_unsupported_content_after_body_and_html
+	 *
+	 * @covers ::next_token
+	 *
+	 * @param string   $html                    Full HTML document.
+	 * @param string[] $breadcrumbs_before_bail Breadcrumbs of the last node visited before bailing.
+	 */
+	public function test_bails_on_unsupported_content_after_body_and_html( string $html, array $breadcrumbs_before_bail ) {
+		$processor = WP_HTML_Processor::create_full_parser( $html );
+
+		$breadcrumbs = array();
+		while ( $processor->next_token() ) {
+			$breadcrumbs = $processor->get_breadcrumbs();
+		}
+
+		$this->assertSame( WP_HTML_Processor::ERROR_UNSUPPORTED, $processor->get_last_error(), 'Failed to bail on unsupported content.' );
+		$this->assertSame( $breadcrumbs_before_bail, $breadcrumbs, 'Bailed at the wrong token.' );
+	}
+
+	/**
+	 * Data provider.
+	 *
+	 * @return array[]
+	 */
+	public static function data_unsupported_content_after_body_and_html(): array {
+		return array(
+			'Whitespace after comment after BODY'          => array( '<html><body></body><!--a--> ', array( 'HTML', '#comment' ) ),
+			'Whitespace after processing instruction after BODY' => array( '<html><body></body><?pi data?> ', array( 'HTML', '#processing-instruction' ) ),
+			'Whitespace after presumptuous tag after BODY' => array( '<html><body></body></> ', array( 'HTML', '#presumptuous-tag' ) ),
+			'Whitespace into unclosed P after comment after BODY' => array( '<p>x</body><!--a--> ', array( 'HTML', '#comment' ) ),
+			'Text after comment after BODY'                => array( '<html><body></body><!--a-->x', array( 'HTML', '#comment' ) ),
+			'NULL byte after comment after BODY'           => array( "<html><body></body><!--a-->\x00", array( 'HTML', '#comment' ) ),
+			'Element after comment after BODY'             => array( '<html><body></body><!--a--><p>', array( 'HTML', '#comment' ) ),
+			'BODY end tag after comment after BODY'        => array( '<html><body></body><!--a--></body>', array( 'HTML', '#comment' ) ),
+			'Whitespace after HTML after comment after BODY' => array( '<html><body></body><!--a--></html> ', array( 'HTML', '#comment' ) ),
+			'Newline after comment after HTML'             => array( "<html><body></body></html><!--a-->\n", array( '#comment' ) ),
+			'Text after comment after HTML'                => array( '<html><body></body></html><!--a-->x', array( '#comment' ) ),
+			'Element after comment after HTML'             => array( '<html><body></body></html><!--a--><p>', array( '#comment' ) ),
+			'HTML end tag after comment after HTML'        => array( '<html><body></body></html><!--a--></html>', array( '#comment' ) ),
+			'Whitespace after comment after FRAMESET and HTML' => array( '<frameset></frameset></html><!--a--> ', array( '#comment' ) ),
+			'NOFRAMES after comment after FRAMESET and HTML' => array( '<frameset></frameset></html><!--a--><noframes></noframes>', array( '#comment' ) ),
+		);
+	}
 }