refactor: harden and isolate link preview fetching

app/Actions/UpdateLinkPreview.php

@@ -4,244 +4,107 @@
 
 namespace App\Actions;
 
-use App\Contracts\Services\DnsResolver;
+use App\Contracts\Services\PreviewLink;
+use App\Dto\LinkPreviewMetadata;
+use App\Dto\LinkPreviewRequest;
 use App\Models\LinkPreview;
-use Dom\HTMLDocument;
-use GuzzleHttp\Exception\ResponseException;
-use GuzzleHttp\Psr7\Utils;
-use Illuminate\Http\Client\ConnectionException;
+use App\Services\LinkPreviews\LinkPreviewAddress;
+use App\Services\LinkPreviews\LinkPreviewBudget;
+use App\Services\LinkPreviews\LinkPreviewHttpClient;
+use App\Services\PreviewLinkProviders\HtmlPreviewLink;
 use Illuminate\Http\Client\Response;
-use Illuminate\Support\Facades\Http;
-use Illuminate\Support\Str;
-use Symfony\Component\HttpFoundation\IpUtils;
 use Uri\WhatWg\Url;
 
-/**
- * @phpstan-type Metadata array{title: string|null, description: string|null, image_url: string|null, site_name: string|null}
- */
 final readonly class UpdateLinkPreview
 {
     private const int MAX_REDIRECTS = 3;
 
-    private const int MAX_BODY_BYTES = 1_048_576;
-
-    private const int MAX_IMAGE_URL_LENGTH = 2048;
-
-    /**
-     * Non-public ranges that filter_var's private and reserved range flags let through: carrier-grade NAT, IETF protocol
-     * assignments, 6to4 relays, benchmarking, documentation and multicast for IPv4; NAT64, discard, Teredo,
-     * documentation, 6to4, site-local, multicast and IPv4-translated addresses for IPv6.
-     *
-     * @var list<string>
-     */
-    private const array BLOCKED_RANGES = [
-        '100.64.0.0/10',
-        '192.0.0.0/24',
-        '192.0.2.0/24',
-        '192.88.99.0/24',
-        '198.18.0.0/15',
-        '198.51.100.0/24',
-        '203.0.113.0/24',
-        '224.0.0.0/4',
-        '64:ff9b::/96',
-        '64:ff9b:1::/48',
-        '100::/64',
-        '2001::/32',
-        '2001:db8::/32',
-        '2002::/16',
-        'fec0::/10',
-        'ff00::/8',
-        '::ffff:0:0:0/96',
-    ];
-
-    private const array NO_METADATA = [
-        'title' => null,
-        'description' => null,
-        'image_url' => null,
-        'site_name' => null,
-    ];
-
-    public function __construct(private DnsResolver $dns)
-    {
+    private const string HEADER_ACCEPT = 'Accept';
+
+    public function __construct(
+        private LinkPreviewAddress $addresses,
+        private LinkPreviewBudget $budget,
+        private LinkPreviewHttpClient $http,
+        private HtmlPreviewLink $html,
+    ) {
         //
     }
 
-    /**
-     * Fetches the page behind the link and stores its Open Graph metadata. A page that can't or mustn't be fetched is
-     * marked as fetched without metadata, so it is not tried again. Failures that may pass, such as connection errors,
-     * timeouts, rate limits and server errors, are thrown instead, so the caller can try again later.
-     */
     public function handle(LinkPreview $linkPreview): void
     {
-        $linkPreview->update([...$this->fetchMetadata($linkPreview->url), 'fetched_at' => now()]);
+        $metadata = $this->fetchMetadata($linkPreview->url);
+
+        $linkPreview->update([...$metadata->toArray(), 'fetched_at' => now()]);
     }
 
-    /**
-     * Follows up to three redirects by hand, so that every hop goes through the same checks as the original URL.
-     *
-     * @return Metadata
-     */
-    private function fetchMetadata(string $url): array
+    private function fetchMetadata(string $url): LinkPreviewMetadata
     {
         $page = Url::parse($url);
+        $deadline = $this->budget->start();
+        $visited = [];
 
         for ($hop = 0; $hop <= self::MAX_REDIRECTS && $page instanceof Url; $hop++) {
-            $address = $this->publicAddress($page);
+            $source = $page->withFragment(null)->toAsciiString();
 
-            if ($address === null) {
-                return self::NO_METADATA;
+            if (isset($visited[$source])) {
+                return new LinkPreviewMetadata;
             }
 
-            $response = $this->get($page, $address);
+            $visited[$source] = true;
+            $request = $this->prepareRequest($page, $deadline);
 
-            if ($response->serverError() || $response->tooManyRequests()) {
-                $response->throw();
+            if (! $request instanceof LinkPreviewRequest) {
+                return new LinkPreviewMetadata;
             }
 
-            if (! $response->redirect()) {
-                return $response->successful() && $this->isHtml($response)
-                    ? $this->parse($this->body($response), $page)
-                    : self::NO_METADATA;
-            }
-
-            $page = Url::parse($response->header('Location'), $page);
-        }
-
-        return self::NO_METADATA;
-    }
+            $response = $this->http->get($request);
+            $response->throwIf($response->serverError() || $response->tooManyRequests());
 
-    /**
-     * Returns the address to connect to, or null when the URL must not be fetched: anything but plain http(s) on a
-     * default port without credentials, or a host that resolves to nothing or to any non-public address.
-     */
-    private function publicAddress(Url $url): ?string
-    {
-        if (! in_array($url->getScheme(), ['http', 'https'], true)
-            || ($url->getUsername() ?? '') !== ''
-            || ($url->getPassword() ?? '') !== ''
-            || ! in_array($url->getPort(), [null, 80, 443], true)) {
-            return null;
-        }
-
-        $addresses = $this->dns->resolve(mb_trim((string) $url->getAsciiHost(), '[]'));
-
-        return $addresses !== [] && array_all($addresses, $this->isPublic(...)) ? $addresses[0] : null;
-    }
-
-    private function isPublic(string $address): bool
-    {
-        return filter_var($address, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE) !== false
-            && ! IpUtils::checkIp($address, self::BLOCKED_RANGES);
-    }
-
-    /**
-     * Connects to the address that was checked rather than resolving the host again, so DNS rebinding can't swap in a
-     * private address. The empty host and port in CURLOPT_CONNECT_TO match every request, so the pin holds even if
-     * cURL were to read the URL differently than we do. Bodies are cut off after 1 MB, and a body that was cut off
-     * (by that limit or by the timeout) is still used, as metadata lives at the top of the page. English is asked for
-     * explicitly, as sites such as YouTube otherwise pick the language from the server's location.
-     */
-    private function get(Url $url, string $address): Response
-    {
-        $port = $url->getPort() ?? ($url->getScheme() === 'https' ? 443 : 80);
-
-        $request = Http::withUserAgent(sprintf('Mozilla/5.0 (compatible; devnull-linkpreview/1.0; +%s)', config()->string('app.url')))
-            ->accept('text/html')
-            ->withHeaders(['Accept-Language' => 'en-US,en;q=0.9'])
-            ->connectTimeout(3)
-            ->timeout(5)
-            ->withoutRedirecting()
-            ->withOptions([
-                'decode_content' => false,
-                'progress' => fn (int $expected, int $downloaded): bool => $downloaded > self::MAX_BODY_BYTES,
-                'curl' => [
-                    CURLOPT_CONNECT_TO => [sprintf('::%s:%d', str_contains($address, ':') ? "[{$address}]" : $address, $port)],
-                ],
-            ]);
-
-        try {
-            return $request->get($url->toAsciiString());
-        } catch (ConnectionException $connectionException) {
-            $cutOff = $connectionException->getPrevious();
-
-            if ($cutOff instanceof ResponseException) {
-                return new Response($cutOff->getResponse());
+            if (! $response->redirect()) {
+                return $this->responseMetadata($response, $page, $this->html);
             }
 
-            throw $connectionException;
+            $page = $this->redirectPage($response, $page);
         }
-    }
-
-    private function isHtml(Response $response): bool
-    {
-        $mediaType = mb_strtolower(mb_trim(Str::before($response->header('Content-Type'), ';')));
-
-        return in_array($mediaType, ['text/html', 'application/xhtml+xml'], true);
-    }
-
-    private function body(Response $response): string
-    {
-        $body = $response->toPsrResponse()->getBody();
-        $body->rewind();
 
-        return Utils::copyToString($body, self::MAX_BODY_BYTES);
+        return new LinkPreviewMetadata;
     }
 
-    /**
-     * @return Metadata
-     */
-    private function parse(string $html, Url $page): array
+    private function prepareRequest(Url $page, float $deadline): ?LinkPreviewRequest
     {
-        $document = HTMLDocument::createFromString($html, LIBXML_NOERROR);
-        $meta = [];
+        $address = $this->addresses->get($page);
+        $remaining = $this->budget->remaining($deadline);
 
-        foreach ($document->querySelectorAll('meta[content]') as $element) {
-            $key = mb_strtolower(mb_trim($element->getAttribute('property') ?? $element->getAttribute('name') ?? ''));
-            $meta[$key] ??= (string) $element->getAttribute('content');
+        if ($address === null || $remaining <= 0) {
+            return null;
         }
 
-        return [
-            'title' => $this->text([$meta['og:title'] ?? '', $meta['twitter:title'] ?? '', $document->title], 300),
-            'description' => $this->text([$meta['og:description'] ?? '', $meta['twitter:description'] ?? '', $meta['description'] ?? ''], 500),
-            'image_url' => $this->imageUrl([$meta['og:image:secure_url'] ?? '', $meta['og:image'] ?? '', $meta['twitter:image'] ?? ''], $page),
-            'site_name' => $this->text([$meta['og:site_name'] ?? ''], 100),
-        ];
+        return new LinkPreviewRequest(
+            url: $page,
+            address: $address,
+            headers: [self::HEADER_ACCEPT => HtmlPreviewLink::CONTENT_TYPE],
+            maxBodyBytes: HtmlPreviewLink::MAX_BODY_BYTES,
+            timeoutSeconds: $remaining,
+        );
     }
 
-    /**
-     * Returns the first candidate that still has text once entities are decoded and whitespace is collapsed,
-     * truncated to the column's length.
-     *
-     * @param  list<string>  $candidates
-     */
-    private function text(array $candidates, int $limit): ?string
+    private function redirectPage(Response $response, Url $page): ?Url
     {
-        foreach ($candidates as $candidate) {
-            $text = Str::squish(html_entity_decode($candidate, ENT_QUOTES | ENT_HTML5, 'UTF-8'));
+        $location = mb_trim($response->header('Location'));
 
-            if ($text !== '') {
-                return mb_strlen($text) > $limit ? mb_rtrim(mb_substr($text, 0, $limit - 1)).'…' : $text;
-            }
+        if ($location === '') {
+            return null;
         }
 
-        return null;
+        return Url::parse($location, $page);
     }
 
-    /**
-     * Returns the first candidate that, resolved against the page URL, is an https URL.
-     *
-     * @param  list<string>  $candidates
-     */
-    private function imageUrl(array $candidates, Url $page): ?string
+    private function responseMetadata(Response $response, Url $page, PreviewLink $previewLink): LinkPreviewMetadata
     {
-        foreach ($candidates as $candidate) {
-            $image = mb_trim($candidate) === '' ? null : Url::parse($candidate, $page);
-
-            if ($image?->getScheme() === 'https' && mb_strlen($image->toAsciiString()) <= self::MAX_IMAGE_URL_LENGTH) {
-                return $image->toAsciiString();
-            }
+        if (! $response->successful()) {
+            return new LinkPreviewMetadata;
         }
 
-        return null;
+        return $previewLink->metadata($response, $page);
     }
 }

app/Contracts/Services/PreviewLink.php

@@ -0,0 +1,14 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Contracts\Services;
+
+use App\Dto\LinkPreviewMetadata;
+use Illuminate\Http\Client\Response;
+use Uri\WhatWg\Url;
+
+interface PreviewLink
+{
+    public function metadata(Response $response, Url $finalUrl): LinkPreviewMetadata;
+}

app/Dto/LinkPreviewMetadata.php

@@ -0,0 +1,35 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Dto;
+
+use Illuminate\Contracts\Support\Arrayable;
+
+/**
+ * @implements Arrayable<string, string|null>
+ */
+final readonly class LinkPreviewMetadata implements Arrayable
+{
+    public function __construct(
+        public ?string $title = null,
+        public ?string $description = null,
+        public ?string $imageUrl = null,
+        public ?string $siteName = null,
+    ) {
+        //
+    }
+
+    /**
+     * @return array{title: string|null, description: string|null, image_url: string|null, site_name: string|null}
+     */
+    public function toArray(): array
+    {
+        return [
+            'title' => $this->title,
+            'description' => $this->description,
+            'image_url' => $this->imageUrl,
+            'site_name' => $this->siteName,
+        ];
+    }
+}

app/Dto/LinkPreviewRequest.php

@@ -0,0 +1,23 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Dto;
+
+use Uri\WhatWg\Url;
+
+final readonly class LinkPreviewRequest
+{
+    /**
+     * @param  array<string, string>  $headers
+     */
+    public function __construct(
+        public Url $url,
+        public string $address,
+        public array $headers,
+        public int $maxBodyBytes,
+        public float $timeoutSeconds,
+    ) {
+        //
+    }
+}