From a71c63f7447275c4c01c8ad3a89cadd867af2772 Mon Sep 17 00:00:00 2001 From: Dennis Snell Date: Thu, 8 Aug 2024 12:26:34 -0700 Subject: [PATCH 1/5] HTML API: Add support for missing FRAMESET and "after" insertion modes. As part of work to add more spec support to the HTML API, this patch adds support for the FRAMESET-related insertion modes, as well as the set of missing _after_ insertion modes. These modes run at the end of parsing a document, closing it and taking care of any lingering tags. Developed in https://github.com/wordpress/wordpress-develop/pull/7165 Discussed in https://core.trac.wordpress.org/ticket/61576 See #61576. --- .../html-api/class-wp-html-processor.php | 321 +++++++++++++++++- .../html-api/wpHtmlProcessorHtml5lib.php | 3 + 2 files changed, 319 insertions(+), 5 deletions(-) diff --git a/src/wp-includes/html-api/class-wp-html-processor.php b/src/wp-includes/html-api/class-wp-html-processor.php index 2bb6302c99781..bd4afa5f1e38b 100644 --- a/src/wp-includes/html-api/class-wp-html-processor.php +++ b/src/wp-includes/html-api/class-wp-html-processor.php @@ -4001,7 +4001,67 @@ private function step_in_template(): bool { * @return bool Whether an element was found. */ private function step_after_body(): bool { - $this->bail( 'No support for parsing in the ' . WP_HTML_Processor_State::INSERTION_MODE_AFTER_BODY . ' state.' ); + $tag_name = $this->get_token_name(); + $token_type = $this->get_token_type(); + $op_sigil = '#tag' === $token_type ? ( $this->is_tag_closer() ? '-' : '+' ) : ''; + $op = "{$op_sigil}{$tag_name}"; + + switch ( $op ) { + /* + * > A character token that is one of U+0009 CHARACTER TABULATION, U+000A LINE FEED (LF), + * > U+000C FORM FEED (FF), U+000D CARRIAGE RETURN (CR), or U+0020 SPACE + * + * > Process the token using the rules for the "in body" insertion mode. + */ + case '#text': + $text = $this->get_modifiable_text(); + if ( strlen( $text ) === strspn( $text, " \t\n\f\r" ) ) { + return $this->step_in_body(); + } + goto after_body_anything_else; + break; + + /* + * > A comment token + */ + case '#cdata-section': + case '#comment': + case '#funky_comment': + $this->insert_html_element( $this->state->current_token ); + return true; + + /* + * > A DOCTYPE token + */ + case 'html': + // Parse error: ignore the token. + return $this->step(); + + /* + * > A start tag whose tag name is "html" + */ + case '+HTML': + return $this->step_in_body(); + + /* + * > An end tag whose tag name is "html" + * + * > If the parser was created as part of the HTML fragment parsing algorithm, + * > this is a parse error; ignore the token. (fragment case) + * > + * > Otherwise, switch the insertion mode to "after after body". + */ + case '-HTML': + if ( isset( $this->context_node ) ) { + return $this->step(); + } + + $this->state->insertion_mode = WP_HTML_Processor_State::INSERTION_MODE_AFTER_AFTER_BODY; + return true; + } + + after_body_anything_else: + $this->bail( 'Cannot process any tokens after closing the BODY which would require re-opening it.' ); } /** @@ -4020,7 +4080,107 @@ private function step_after_body(): bool { * @return bool Whether an element was found. */ private function step_in_frameset(): bool { - $this->bail( 'No support for parsing in the ' . WP_HTML_Processor_State::INSERTION_MODE_IN_FRAMESET . ' state.' ); + $tag_name = $this->get_token_name(); + $token_type = $this->get_token_type(); + $op_sigil = '#tag' === $token_type ? ( $this->is_tag_closer() ? '-' : '+' ) : ''; + $op = "{$op_sigil}{$tag_name}"; + + switch ( $op ) { + /* + * > A character token that is one of U+0009 CHARACTER TABULATION, U+000A LINE FEED (LF), + * > U+000C FORM FEED (FF), U+000D CARRIAGE RETURN (CR), or U+0020 SPACE + * + * > Insert the character. + */ + case '#text': + $text = $this->get_modifiable_text(); + if ( strlen( $text ) === strspn( $text, " \t\n\f\r" ) ) { + $this->insert_html_element( $this->state->current_token ); + return true; + } + goto in_frameset_anything_else; + break; + + /* + * > A comment token + */ + case '#cdata-section': + case '#comment': + case '#funky_comment': + $this->insert_html_element( $this->state->current_token ); + return true; + + /* + * > A DOCTYPE token + */ + case 'html': + // Parse error: ignore the token. + return $this->step(); + + /* + * > A start tag whose tag name is "html" + */ + case '+HTML': + return $this->step_in_body(); + + /* + * > A start tag whose tag name is "frameset" + */ + case '+FRAMESET': + $this->insert_html_element( $this->state->current_token ); + return true; + + /* + * > An end tag whose tag name is "frameset" + */ + case '-FRAMESET': + /* + * > If the current node is the root html element, then this is a parse error; + * > ignore the token. (fragment case) + */ + if ( $this->state->stack_of_open_elements->current_node_is( 'HTML' ) ) { + return $this->step(); + } + + /* + * > Otherwise, pop the current node from the stack of open elements. + */ + $this->state->stack_of_open_elements->pop(); + + /* + * > If the parser was not created as part of the HTML fragment parsing algorithm + * > (fragment case), and the current node is no longer a frameset element, then + * > switch the insertion mode to "after frameset". + */ + if ( ! isset( $this->context_node ) && ! $this->state->stack_of_open_elements->current_node_is( 'FRAMESET' ) ) { + $this->state->insertion_mode = WP_HTML_Processor_State::INSERTION_MODE_AFTER_FRAMESET; + } + + return true; + + /* + * > A start tag whose tag name is "frame" + * + * > Insert an HTML element for the token. Immediately pop the + * > current node off the stack of open elements. + * > + * > Acknowledge the token's self-closing flag, if it is set. + */ + case '+FRAME': + $this->insert_html_element( $this->state->current_token ); + $this->state->stack_of_open_elements->pop(); + return true; + + /* + * > A start tag whose tag name is "noframes" + */ + case '+NOFRAMES': + return $this->step_in_head(); + } + + in_frameset_anything_else: + // Parse error: ignore the token. + return $this->step(); } /** @@ -4039,7 +4199,66 @@ private function step_in_frameset(): bool { * @return bool Whether an element was found. */ private function step_after_frameset(): bool { - $this->bail( 'No support for parsing in the ' . WP_HTML_Processor_State::INSERTION_MODE_AFTER_FRAMESET . ' state.' ); + $tag_name = $this->get_token_name(); + $token_type = $this->get_token_type(); + $op_sigil = '#tag' === $token_type ? ( $this->is_tag_closer() ? '-' : '+' ) : ''; + $op = "{$op_sigil}{$tag_name}"; + + switch ( $op ) { + /* + * > A character token that is one of U+0009 CHARACTER TABULATION, U+000A LINE FEED (LF), + * > U+000C FORM FEED (FF), U+000D CARRIAGE RETURN (CR), or U+0020 SPACE + * + * > Insert the character. + */ + case '#text': + $text = $this->get_modifiable_text(); + if ( strlen( $text ) === strspn( $text, " \t\n\f\r" ) ) { + $this->insert_html_element( $this->state->current_token ); + return true; + } + goto after_frameset_anything_else; + break; + + /* + * > A comment token + */ + case '#cdata-section': + case '#comment': + case '#funky_comment': + $this->insert_html_element( $this->state->current_token ); + return true; + + /* + * > A DOCTYPE token + */ + case 'html': + // Parse error: ignore the token. + return $this->step(); + + /* + * > A start tag whose tag name is "html" + */ + case '+HTML': + return $this->step_in_body(); + + /* + * > An end tag whose tag name is "html" + */ + case '-HTML': + $this->state->insertion_mode = WP_HTML_Processor_State::INSERTION_MODE_AFTER_AFTER_FRAMESET; + return true; + + /* + * > A start tag whose tag name is "noframes" + */ + case '+NOFRAMES': + return $this->step_in_head(); + } + + after_frameset_anything_else: + // Parse error: ignore the token. + return $this->step(); } /** @@ -4058,7 +4277,51 @@ private function step_after_frameset(): bool { * @return bool Whether an element was found. */ private function step_after_after_body(): bool { - $this->bail( 'No support for parsing in the ' . WP_HTML_Processor_State::INSERTION_MODE_AFTER_AFTER_BODY . ' state.' ); + $tag_name = $this->get_token_name(); + $token_type = $this->get_token_type(); + $op_sigil = '#tag' === $token_type ? ( $this->is_tag_closer() ? '-' : '+' ) : ''; + $op = "{$op_sigil}{$tag_name}"; + + switch ( $op ) { + /* + * > A comment token + */ + case '#cdata-section': + case '#comment': + case '#funky_comment': + $this->insert_html_element( $this->state->current_token ); + return true; + + /* + * > A DOCTYPE token + * > A start tag whose tag name is "html" + * + * > Process the token using the rules for the "in body" insertion mode. + */ + case 'html': + case '+HTML': + return $this->step_in_body(); + + /* + * > A character token that is one of U+0009 CHARACTER TABULATION, U+000A LINE FEED (LF), + * > U+000C FORM FEED (FF), U+000D CARRIAGE RETURN (CR), or U+0020 SPACE + * + * > Process the token using the rules for the "in body" insertion mode. + */ + case '#text': + $text = $this->get_modifiable_text(); + if ( strlen( $text ) === strspn( $text, " \t\n\f\r" ) ) { + return $this->step_in_body(); + } + goto after_after_body_anything_else; + break; + } + + /* + * > Parse error. Switch the insertion mode to "in body" and reprocess the token. + */ + after_after_body_anything_else: + $this->bail( 'Cannot process any tokens after closing the BODY which would require re-opening it.' ); } /** @@ -4077,7 +4340,55 @@ private function step_after_after_body(): bool { * @return bool Whether an element was found. */ private function step_after_after_frameset(): bool { - $this->bail( 'No support for parsing in the ' . WP_HTML_Processor_State::INSERTION_MODE_AFTER_AFTER_FRAMESET . ' state.' ); + $tag_name = $this->get_token_name(); + $token_type = $this->get_token_type(); + $op_sigil = '#tag' === $token_type ? ( $this->is_tag_closer() ? '-' : '+' ) : ''; + $op = "{$op_sigil}{$tag_name}"; + + switch ( $op ) { + /* + * > A comment token + */ + case '#cdata-section': + case '#comment': + case '#funky_comment': + $this->insert_html_element( $this->state->current_token ); + return true; + + /* + * > A DOCTYPE token + * > A start tag whose tag name is "html" + * + * > Process the token using the rules for the "in body" insertion mode. + */ + case 'html': + case '+HTML': + return $this->step_in_body(); + + /* + * > A character token that is one of U+0009 CHARACTER TABULATION, U+000A LINE FEED (LF), + * > U+000C FORM FEED (FF), U+000D CARRIAGE RETURN (CR), or U+0020 SPACE + * + * > Process the token using the rules for the "in body" insertion mode. + */ + case '#text': + $text = $this->get_modifiable_text(); + if ( strlen( $text ) === strspn( $text, " \t\n\f\r" ) ) { + return $this->step_in_body(); + } + goto after_after_frameset_anything_else; + break; + + /* + * > A start tag whose tag name is "noframes" + */ + case '+NOFRAMES': + return $this->step_in_head(); + } + + after_after_frameset_anything_else: + // Parse error: ignore the token. + return $this->step(); } /** diff --git a/tests/phpunit/tests/html-api/wpHtmlProcessorHtml5lib.php b/tests/phpunit/tests/html-api/wpHtmlProcessorHtml5lib.php index 4de4ebd1cd5c4..66e3791f58fbe 100644 --- a/tests/phpunit/tests/html-api/wpHtmlProcessorHtml5lib.php +++ b/tests/phpunit/tests/html-api/wpHtmlProcessorHtml5lib.php @@ -32,10 +32,13 @@ class Tests_HtmlApi_Html5lib extends WP_UnitTestCase { 'tests1/line0692' => 'Bug: Mixed whitespace, non-whitespace text in head not split correctly', 'tests14/line0022' => 'Unimplemented: This parser does not add missing attributes to existing HTML or BODY tags.', 'tests14/line0055' => 'Unimplemented: This parser does not add missing attributes to existing HTML or BODY tags.', + 'tests19/line0488' => 'Unimplemented: This parser does not add missing attributes to existing HTML or BODY tags.', + 'tests19/line0500' => 'Unimplemented: This parser does not add missing attributes to existing HTML or BODY tags.', 'tests19/line0965' => 'Bug: Mixed whitespace, non-whitespace text in head not split correctly.', 'tests19/line1079' => 'Unimplemented: This parser does not add missing attributes to existing HTML or BODY tags.', 'tests2/line0207' => 'Unimplemented: This parser does not add missing attributes to existing HTML or BODY tags.', 'tests2/line0686' => 'Unimplemented: This parser does not add missing attributes to existing HTML or BODY tags.', + 'tests2/line0697' => 'Unimplemented: This parser does not add missing attributes to existing HTML or BODY tags.', 'tests2/line0709' => 'Unimplemented: This parser does not add missing attributes to existing HTML or BODY tags.', 'tests5/line0013' => 'Bug: Mixed whitespace, non-whitespace text in head not split correctly.', 'tests5/line0077' => 'Bug: Mixed whitespace, non-whitespace text in head not split correctly.', From 0dd9189c01d71fca9a9a232fd2a0233abbc227de Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 9 Aug 2024 10:01:34 +0200 Subject: [PATCH 2/5] Fix #funky_comment token name (#funky-comment) --- src/wp-includes/html-api/class-wp-html-processor.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/wp-includes/html-api/class-wp-html-processor.php b/src/wp-includes/html-api/class-wp-html-processor.php index bd4afa5f1e38b..a020fb0316ca6 100644 --- a/src/wp-includes/html-api/class-wp-html-processor.php +++ b/src/wp-includes/html-api/class-wp-html-processor.php @@ -4026,7 +4026,7 @@ private function step_after_body(): bool { */ case '#cdata-section': case '#comment': - case '#funky_comment': + case '#funky-comment': $this->insert_html_element( $this->state->current_token ); return true; @@ -4106,7 +4106,7 @@ private function step_in_frameset(): bool { */ case '#cdata-section': case '#comment': - case '#funky_comment': + case '#funky-comment': $this->insert_html_element( $this->state->current_token ); return true; @@ -4225,7 +4225,7 @@ private function step_after_frameset(): bool { */ case '#cdata-section': case '#comment': - case '#funky_comment': + case '#funky-comment': $this->insert_html_element( $this->state->current_token ); return true; @@ -4288,7 +4288,7 @@ private function step_after_after_body(): bool { */ case '#cdata-section': case '#comment': - case '#funky_comment': + case '#funky-comment': $this->insert_html_element( $this->state->current_token ); return true; @@ -4351,7 +4351,7 @@ private function step_after_after_frameset(): bool { */ case '#cdata-section': case '#comment': - case '#funky_comment': + case '#funky-comment': $this->insert_html_element( $this->state->current_token ); return true; @@ -4455,7 +4455,7 @@ private function step_in_foreign_content(): bool { */ case '#cdata-section': case '#comment': - case '#funky_comment': + case '#funky-comment': $this->insert_foreign_element( $this->state->current_token, false ); return true; From c74093207e5eca9b66b0f434a5f59ee9e249f007 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 9 Aug 2024 11:01:38 +0200 Subject: [PATCH 3/5] Add presumptuous tag handling to after(after) body --- src/wp-includes/html-api/class-wp-html-processor.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/wp-includes/html-api/class-wp-html-processor.php b/src/wp-includes/html-api/class-wp-html-processor.php index a020fb0316ca6..e11917c2ca4fc 100644 --- a/src/wp-includes/html-api/class-wp-html-processor.php +++ b/src/wp-includes/html-api/class-wp-html-processor.php @@ -4027,6 +4027,7 @@ private function step_after_body(): bool { case '#cdata-section': case '#comment': case '#funky-comment': + case '#presumptuous-tag': $this->insert_html_element( $this->state->current_token ); return true; @@ -4289,6 +4290,7 @@ private function step_after_after_body(): bool { case '#cdata-section': case '#comment': case '#funky-comment': + case '#presumptuous-tag': $this->insert_html_element( $this->state->current_token ); return true; From 6d5fbf29258337ed8973162298bedb9b2506f38b Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 9 Aug 2024 11:01:55 +0200 Subject: [PATCH 4/5] Add missing spec quote to after body anything else --- src/wp-includes/html-api/class-wp-html-processor.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/wp-includes/html-api/class-wp-html-processor.php b/src/wp-includes/html-api/class-wp-html-processor.php index e11917c2ca4fc..6911812761c08 100644 --- a/src/wp-includes/html-api/class-wp-html-processor.php +++ b/src/wp-includes/html-api/class-wp-html-processor.php @@ -4061,6 +4061,9 @@ private function step_after_body(): bool { return true; } + /* + * > Parse error. Switch the insertion mode to "in body" and reprocess the token. + */ after_body_anything_else: $this->bail( 'Cannot process any tokens after closing the BODY which would require re-opening it.' ); } From bf214a7e623292f63082a6440e717361863826bc Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 9 Aug 2024 13:33:11 +0200 Subject: [PATCH 5/5] Broken idea for after body comment handling --- .../html-api/class-wp-html-processor.php | 63 ++++++++++++++----- .../html-api/class-wp-html-token.php | 11 ++++ 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/src/wp-includes/html-api/class-wp-html-processor.php b/src/wp-includes/html-api/class-wp-html-processor.php index 6911812761c08..da5b2f78b8252 100644 --- a/src/wp-includes/html-api/class-wp-html-processor.php +++ b/src/wp-includes/html-api/class-wp-html-processor.php @@ -256,6 +256,16 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { */ private $context_node = null; + /** + * @var WP_HTML_Token[] + */ + private $after_body_comments = array(); + + /** + * @var WP_HTML_Token[] + */ + private $after_after_body_comments = array(); + /* * Public Interface Functions */ @@ -639,12 +649,34 @@ public function next_token(): bool { // Process the next event on the queue. $this->current_element = array_shift( $this->element_queue ); if ( ! isset( $this->current_element ) ) { - // There are no tokens left, so close all remaining open elements. + // The document has been fully processed. Perform final actions. + + // Close out the body: + if ( $this->state->stack_of_open_elements->contains( 'BODY' ) ) { + $this->state->stack_of_open_elements->pop_until( 'BODY' ); + } + + // Deal with after body comments: + foreach ( $this->after_body_comments as $token ) { + $this->insert_html_element( $token ); + } + $this->after_body_comments = array(); + + // Close out everything on the stack of open elements: while ( $this->state->stack_of_open_elements->pop() ) { continue; } + // Deal with after after body comments: + foreach ( $this->after_after_body_comments as $token ) { + $this->insert_html_element( $token ); + } + $this->after_after_body_comments = array(); + + if ( ! empty( $this->element_queue ) ) { + return $this->next_token(); + } - return empty( $this->element_queue ) ? false : $this->next_token(); + return false; } $is_pop = WP_HTML_Stack_Event::POP === $this->current_element->operation; @@ -868,6 +900,9 @@ public function step( $node_to_process = self::PROCESS_NEXT_NODE ): bool { $this->has_self_closing_flag(), $this->release_internal_bookmark_on_destruct ); + if ( '#comment' === $token_name ) { + $this->state->current_token->comment_type = $this->get_comment_type(); + } } $parse_in_current_insertion_mode = ( @@ -4028,8 +4063,8 @@ private function step_after_body(): bool { case '#comment': case '#funky-comment': case '#presumptuous-tag': - $this->insert_html_element( $this->state->current_token ); - return true; + $this->after_body_comments[] = $this->state->current_token; + return $this->step(); /* * > A DOCTYPE token @@ -4065,7 +4100,8 @@ private function step_after_body(): bool { * > Parse error. Switch the insertion mode to "in body" and reprocess the token. */ after_body_anything_else: - $this->bail( 'Cannot process any tokens after closing the BODY which would require re-opening it.' ); + $this->state->insertion_mode = WP_HTML_Processor_State::INSERTION_MODE_IN_BODY; + return $this->step( self::REPROCESS_CURRENT_NODE ); } /** @@ -4294,8 +4330,8 @@ private function step_after_after_body(): bool { case '#comment': case '#funky-comment': case '#presumptuous-tag': - $this->insert_html_element( $this->state->current_token ); - return true; + $this->after_after_body_comments[] = $this->state->current_token; + return $this->step(); /* * > A DOCTYPE token @@ -4326,7 +4362,8 @@ private function step_after_after_body(): bool { * > Parse error. Switch the insertion mode to "in body" and reprocess the token. */ after_after_body_anything_else: - $this->bail( 'Cannot process any tokens after closing the BODY which would require re-opening it.' ); + $this->state->insertion_mode = WP_HTML_Processor_State::INSERTION_MODE_IN_BODY; + return $this->step( self::REPROCESS_CURRENT_NODE ); } /** @@ -4759,7 +4796,7 @@ public function get_tag(): ?string { return null; } - if ( $this->is_virtual() ) { + if ( $this->current_element ) { return $this->current_element->token->node_name; } @@ -4820,9 +4857,7 @@ public function has_self_closing_flag(): bool { * @return string|null Name of the matched token. */ public function get_token_name(): ?string { - return $this->is_virtual() - ? $this->current_element->token->node_name - : parent::get_token_name(); + return $this->current_element->token->node_name ?? parent::get_token_name(); } /** @@ -4848,7 +4883,7 @@ public function get_token_name(): ?string { * @return string|null What kind of token is matched, or null. */ public function get_token_type(): ?string { - if ( $this->is_virtual() ) { + if ( $this->current_element ) { /* * This logic comes from the Tag Processor. * @@ -5055,7 +5090,7 @@ public function get_modifiable_text(): string { * @return string|null */ public function get_comment_type(): ?string { - return $this->is_virtual() ? null : parent::get_comment_type(); + return $this->current_element->token->comment_type ?? parent::get_comment_type(); } /** diff --git a/src/wp-includes/html-api/class-wp-html-token.php b/src/wp-includes/html-api/class-wp-html-token.php index d5e51ac29007f..4bfa9eb9e6ada 100644 --- a/src/wp-includes/html-api/class-wp-html-token.php +++ b/src/wp-includes/html-api/class-wp-html-token.php @@ -46,6 +46,17 @@ class WP_HTML_Token { */ public $node_name = null; + /** + * Comment type + * + * @since 6.7.0 + * + * @see WP_HTML_Processor::get_comment_type() + * + * @var string|null + */ + public $comment_type = null; + /** * Whether node contains the self-closing flag. *