feature symfony#61475 [DomCrawler] Use the native HTML5 parser on PHP 8.4 (nicolas-grekas)

This PR was merged into the 7.4 branch.

Discussion
----------

[DomCrawler] Use the native HTML5 parser on PHP 8.4

| Q             | A
| ------------- | ---
| Branch?       | 7.4
| Bug fix?      | no
| New feature?  | no
| Deprecations? | no
| Issues        | -
| License       | MIT

This PR keeps the `DOM*`-based API but uses the native HTML5 parser on PHP 8.4 instead of masterminds/html5.
This works by parsing HTML strings using `Dom\HTMLDocument` then serializing to XML, and loading again using `DOMDocument::loadXML()`.

This basically replaces symfony#61356 since it removes any BC breaks.

The drawback compared to a more native approach is the double-parsing that happens.
This could be worked on later by providing a way to leverage the new `Dom\*` API directly.
To be proved worth it before.

Commits
-------

bd7fb51 [DomCrawler] Use the native HTM5 parser on PHP 8.4

Author: nicolas-grekas

UPGRADE-7.4.md

@@ -8,6 +8,11 @@ Read more about this in the [Symfony documentation](https://symfony.com/doc/7.4/
 
 If you're upgrading from a version below 7.3, follow the [7.3 upgrade guide](UPGRADE-7.3.md) first.
 
+BrowserKit
+----------
+
+ * Deprecate `AbstractBrowser::useHtml5Parser()`; Symfony 8 will unconditionally use the native HTML5 parser
+
 Cache
 -----
 
@@ -29,6 +34,11 @@ DoctrineBridge
 
  * Deprecate `UniqueEntity::getRequiredOptions()` and `UniqueEntity::getDefaultOption()`
 
+DomCrawler
+----------
+
+ * Disabling HTML5 parsing is deprecated; Symfony 8 will unconditionally use the native HTML5 parser
+
 FrameworkBundle
 ---------------
 

src/Symfony/Component/BrowserKit/AbstractBrowser.php

@@ -46,6 +46,7 @@ abstract class AbstractBrowser
     /** @psalm-var TResponse */
     protected object $response;
     protected Crawler $crawler;
+    /** @deprecated since Symfony 7.4, to be removed in Symfony 8 */
     protected bool $useHtml5Parser = true;
     protected bool $insulated = false;
     protected ?string $redirect;
@@ -204,10 +205,16 @@ public function getCrawler(): Crawler
     /**
      * Sets whether parsing should be done using "masterminds/html5".
      *
+     * @deprecated since Symfony 7.4, Symfony 8 will unconditionally use the native HTML5 parser
+     *
      * @return $this
      */
     public function useHtml5Parser(bool $useHtml5Parser): static
     {
+        if (\PHP_VERSION_ID >= 80400) {
+            trigger_deprecation('symfony/browser-kit', '7.4', 'Method "%s()" is deprecated. Symfony 8 will unconditionally use the native HTML5 parser.', __METHOD__);
+        }
+
         $this->useHtml5Parser = $useHtml5Parser;
 
         return $this;

src/Symfony/Component/BrowserKit/CHANGELOG.md

@@ -6,6 +6,7 @@ CHANGELOG
 
  * Add `isFirstPage()` and `isLastPage()` methods to the History class for checking navigation boundaries
  * Add PHPUnit constraints: `BrowserHistoryIsOnFirstPage` and `BrowserHistoryIsOnLastPage`
+ * Deprecate `AbstractBrowser::useHtml5Parser()`; Symfony 8 will unconditionally use the native HTML5 parser
 
 6.4
 ---

src/Symfony/Component/BrowserKit/composer.json

@@ -17,6 +17,7 @@
     ],
     "require": {
         "php": ">=8.2",
+        "symfony/deprecation-contracts": "^2.5|^3",
         "symfony/dom-crawler": "^6.4|^7.0|^8.0"
     },
     "require-dev": {

src/Symfony/Component/DomCrawler/CHANGELOG.md

@@ -1,6 +1,11 @@
 CHANGELOG
 =========
 
+7.4
+---
+
+ * Disabling HTML5 parsing is deprecated; Symfony 8 will unconditionally use the native HTML5 parser
+
 7.0
 ---
 

src/Symfony/Component/DomCrawler/Crawler.php

@@ -62,10 +62,14 @@ public function __construct(
         \DOMNodeList|\DOMNode|array|string|null $node = null,
         protected ?string $uri = null,
         ?string $baseHref = null,
-        bool $useHtml5Parser = true,
+        private bool $useHtml5Parser = true,
     ) {
+        if (\PHP_VERSION_ID >= 80400 && !$useHtml5Parser) {
+            trigger_deprecation('symfony/dom-crawler', '7.4', 'Disabling HTML5 parsing is deprecated. Symfony 8 will unconditionally use the native HTML5 parser.');
+        }
+
         $this->baseHref = $baseHref ?: $uri;
-        $this->html5Parser = $useHtml5Parser ? new HTML5(['disable_html_ns' => true]) : null;
+        $this->html5Parser = \PHP_VERSION_ID < 80400 && $useHtml5Parser ? new HTML5(['disable_html_ns' => true]) : null;
         $this->cachedNamespaces = new \ArrayObject();
 
         $this->add($node);
@@ -1081,23 +1085,41 @@ private function supportsEncoding(string $encoding): bool
 
     private function parseXhtml(string $htmlContent, string $charset = 'UTF-8'): \DOMDocument
     {
-        if ('UTF-8' === $charset && preg_match('//u', $htmlContent)) {
-            $htmlContent = '<?xml encoding="UTF-8">'.$htmlContent;
-        } else {
-            $htmlContent = $this->convertToHtmlEntities($htmlContent, $charset);
+        if (\PHP_VERSION_ID < 80400 || !$this->useHtml5Parser) {
+            if ('UTF-8' === $charset && preg_match('//u', $htmlContent)) {
+                $htmlContent = '<?xml encoding="UTF-8">'.$htmlContent;
+            } else {
+                $htmlContent = $this->convertToHtmlEntities($htmlContent, $charset);
+            }
+
+            $internalErrors = libxml_use_internal_errors(true);
+
+            $dom = new \DOMDocument('1.0', $charset);
+            $dom->validateOnParse = true;
+
+            if ('' !== trim($htmlContent)) {
+                @$dom->loadHTML($htmlContent);
+            }
+
+            libxml_use_internal_errors($internalErrors);
+
+            return $dom;
         }
 
-        $internalErrors = libxml_use_internal_errors(true);
+        $document = @\Dom\HTMLDocument::createFromString($htmlContent, \Dom\HTML_NO_DEFAULT_NS, $charset);
+        $htmlContent = $document->saveXml();
+        $charset = $document->inputEncoding;
 
         $dom = new \DOMDocument('1.0', $charset);
-        $dom->validateOnParse = true;
+        $dom->loadXML($htmlContent);
 
-        if ('' !== trim($htmlContent)) {
-            @$dom->loadHTML($htmlContent);
+        // Register id attributes as ID attributes for getElementById to work
+        foreach ((new \DOMXPath($dom))->query('//*[@id]') as $element) {
+            if ($element instanceof \DOMElement) {
+                $element->setIdAttribute('id', true);
+            }
         }
 
-        libxml_use_internal_errors($internalErrors);
-
         return $dom;
     }
 
@@ -1216,7 +1238,7 @@ private function canParseHtml5String(string $content): bool
             return false;
         }
 
-        if (false === ($pos = stripos($content, '<!doctype html>'))) {
+        if (false === $pos = stripos($content, '<!doctype html>')) {
             return false;
         }
 

src/Symfony/Component/DomCrawler/Tests/AbstractCrawlerTestCase.php renamed to src/Symfony/Component/DomCrawler/Tests/CrawlerTestCase.php

@@ -12,6 +12,8 @@
 namespace Symfony\Component\DomCrawler\Tests;
 
 use PHPUnit\Framework\Attributes\DataProvider;
+use PHPUnit\Framework\Attributes\Group;
+use PHPUnit\Framework\Attributes\IgnoreDeprecations;
 use PHPUnit\Framework\Attributes\RequiresPhpExtension;
 use PHPUnit\Framework\Error\Notice;
 use PHPUnit\Framework\TestCase;
@@ -20,21 +22,24 @@
 use Symfony\Component\DomCrawler\Image;
 use Symfony\Component\DomCrawler\Link;
 
-abstract class AbstractCrawlerTestCase extends TestCase
+class CrawlerTestCase extends TestCase
 {
-    abstract public static function getDoctype(): string;
+    public static function getDoctype(): string
+    {
+        return '<!DOCTYPE html>';
+    }
 
-    protected function createCrawler($node = null, ?string $uri = null, ?string $baseHref = null, bool $useHtml5Parser = true)
+    protected function createCrawler($node = null, ?string $uri = null, ?string $baseHref = null)
     {
-        return new Crawler($node, $uri, $baseHref, $useHtml5Parser);
+        return new Crawler($node, $uri, $baseHref, \PHP_VERSION_ID >= 80400);
     }
 
     public function testConstructor()
     {
         $crawler = $this->createCrawler();
         $this->assertCount(0, $crawler, '__construct() returns an empty crawler');
 
-        $doc = new \DOMDocument();
+        $doc = $this->createDomDocument();
         $node = $doc->createElement('test');
 
         $crawler = $this->createCrawler($node);
@@ -236,7 +241,7 @@ public function testAddNode()
 
     public function testClear()
     {
-        $doc = new \DOMDocument();
+        $doc = $this->createDomDocument();
         $node = $doc->createElement('test');
 
         $crawler = $this->createCrawler($node);
@@ -407,7 +412,7 @@ public function testInnerText(
     public function testHtml()
     {
         $this->assertEquals('<img alt="Bar">', $this->createTestCrawler()->filterXPath('//a[5]')->html());
-        $this->assertEquals('<input type="text" value="TextValue" name="TextName"><input type="submit" value="FooValue" name="FooName" id="FooId"><input type="button" value="BarValue" name="BarName" id="BarId"><button value="ButtonValue" name="ButtonName" id="ButtonId"></button>', trim(preg_replace('~>\s+<~', '><', $this->createTestCrawler()->filterXPath('//form[@id="FooFormId"]')->html())));
+        $this->assertEquals('<input type="text" value="TextValue" name="TextName"><input type="submit" value="FooValue" name="FooName" id="FooId"><input type="button" value="BarValue" name="BarName" id="BarId"><button value="ButtonValue" name="ButtonName" id="ButtonId"><input type="submit" value="FooBarValue" name="FooBarName" form="FooFormId"><input type="text" value="FooTextValue" name="FooTextName" form="FooFormId"><input type="image" alt="ImageAlt" form="FooFormId"></button>', trim(preg_replace('~>\s+<~', '><', $this->createTestCrawler()->filterXPath('//form[@id="FooFormId"]')->html())));
 
         try {
             $this->createTestCrawler()->filterXPath('//ol')->html();
@@ -421,9 +426,9 @@ public function testHtml()
 
     public function testEmojis()
     {
-        $crawler = $this->createCrawler('<body><p>Hey 👋</p></body>');
+        $crawler = $this->createCrawler('<head></head><body><p>Hey 👋</p></body>');
 
-        $this->assertSame('<body><p>Hey 👋</p></body>', $crawler->html());
+        $this->assertSame('<head></head><body><p>Hey 👋</p></body>', $crawler->html());
     }
 
     public function testExtract()
@@ -448,7 +453,7 @@ public function testFilterXpathComplexQueries()
         $this->assertCount(1, $crawler->filterXPath('./body'));
         $this->assertCount(1, $crawler->filterXPath('.//body'));
         $this->assertCount(6, $crawler->filterXPath('.//input'));
-        $this->assertCount(4, $crawler->filterXPath('//form')->filterXPath('//button | //input'));
+        $this->assertCount(7, $crawler->filterXPath('//form')->filterXPath('//button | //input'));
         $this->assertCount(1, $crawler->filterXPath('body'));
         $this->assertCount(8, $crawler->filterXPath('//button | //input'));
         $this->assertCount(1, $crawler->filterXPath('//body'));
@@ -530,6 +535,16 @@ public function testFilterXPathWithAnUrl()
         $this->assertSame('Music', $crawler->text());
     }
 
+    public function testCaseSentivity()
+    {
+        $crawler = $this->createTestXmlCrawler();
+
+        $crawler = $crawler->filterXPath('//*[local-name() = "CaseSensitiveTag"]');
+        $this->assertCount(1, $crawler);
+        $this->assertSame('Some Content', $crawler->text());
+        $this->assertSame('CaseSensitiveTag', $crawler->nodeName());
+    }
+
     public function testFilterXPathWithFakeRoot()
     {
         $crawler = $this->createTestCrawler();
@@ -1290,10 +1305,82 @@ public function testAddHtmlContentUnsupportedCharset()
         $this->assertEquals('Žťčýů', $crawler->filterXPath('//p')->text());
     }
 
-    public function createTestCrawler($uri = null)
+    public function testAddXmlContentWithErrors()
     {
-        $dom = new \DOMDocument();
-        $dom->loadHTML($this->getDoctype().'
+        $internalErrors = libxml_use_internal_errors(true);
+
+        $crawler = $this->createCrawler();
+        $crawler->addXmlContent(<<<'EOF'
+            <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+            <html>
+                <head>
+                </head>
+                <body>
+                    <nav><a href="#"><a href="#"></nav>
+                </body>
+            </html>
+            EOF,
+            'UTF-8'
+        );
+
+        $this->assertGreaterThan(1, libxml_get_errors());
+
+        libxml_clear_errors();
+        libxml_use_internal_errors($internalErrors);
+    }
+
+    #[IgnoreDeprecations]
+    #[Group('legacy')]
+    public function testHtml5ParserNotSameAsNativeParserForSpecificHtml()
+    {
+        // Html who create a bug specific to the DOM extension (see https://github.com/symfony/symfony/issues/28596)
+        $html = '<!DOCTYPE html><html><body><h1><p>Foo</p></h1></body></html>';
+
+        $html5Crawler = new Crawler(null, null, null, true);
+        $html5Crawler->add($html);
+
+        $nativeCrawler = new Crawler(null, null, null, false);
+        $nativeCrawler->add($html);
+
+        $this->assertNotEquals($nativeCrawler->filterXPath('//h1')->text(), $html5Crawler->filterXPath('//h1')->text(), 'Native parser and Html5 parser must be different');
+    }
+
+    public function testAddHtml5()
+    {
+        // Ensure a bug specific to the DOM extension is fixed (see https://github.com/symfony/symfony/issues/28596)
+        $crawler = $this->createCrawler();
+        $crawler->add($this->getDoctype().'<html><body><h1><p>Foo</p></h1></body></html>');
+        $this->assertEquals('Foo', $crawler->filterXPath('//h1')->text(), '->add() adds nodes from a string');
+    }
+
+    #[DataProvider('html5Provider')]
+    public function testHtml5ParserParseContentStartingWithValidHeading(string $content)
+    {
+        $crawler = $this->createCrawler();
+        $crawler->addHtmlContent($content);
+        self::assertEquals(
+            'Foo',
+            $crawler->filterXPath('//h1')->text(),
+            '->addHtmlContent() parses valid HTML with comment before doctype'
+        );
+    }
+
+    public static function html5Provider(): iterable
+    {
+        $html = self::getDoctype().'<html><body><h1><p>Foo</p></h1></body></html>';
+        $BOM = \chr(0xEF).\chr(0xBB).\chr(0xBF);
+
+        yield 'BOM first' => [$BOM.$html];
+        yield 'Single comment' => ['<!-- comment -->'.$html];
+        yield 'Multiline comment' => ["<!-- \n multiline comment \n -->".$html];
+        yield 'Several comments' => ['<!--c--> <!--cc-->'.$html];
+        yield 'Whitespaces' => ['    '.$html];
+        yield 'All together' => [$BOM.'  <!--c-->'.$html];
+    }
+
+    protected function createTestCrawler($uri = null)
+    {
+        $html = $this->getDoctype().'
             <html>
                 <body>
                     <a href="foo">Foo</a>
@@ -1352,9 +1439,9 @@ public function createTestCrawler($uri = null)
                     </div>
                 </body>
             </html>
-        ');
+        ';
 
-        return $this->createCrawler($dom, $uri);
+        return $this->createCrawler($html, $uri);
     }
 
     protected function createTestXmlCrawler($uri = null)
@@ -1369,6 +1456,7 @@ protected function createTestXmlCrawler($uri = null)
                     <yt:aspectRatio>widescreen</yt:aspectRatio>
                 </media:group>
                 <media:category label="Music" scheme="http://gdata.youtube.com/schemas/2007/categories.cat">Music</media:category>
+                <CaseSensitiveTag>Some Content</CaseSensitiveTag>
             </entry>';
 
         return $this->createCrawler($xml, $uri);