fix(http): keep proxy credentials out of HTTP crawler errors

crawl4ai/async_crawler_strategy.py

@@ -13,6 +13,7 @@
 from PIL import Image, ImageDraw, ImageFont
 import hashlib
 import random
+import re
 import uuid
 from .js_snippet import load_js_script
 from .models import AsyncCrawlResponse
@@ -2650,6 +2651,42 @@ def _format_proxy_url(self, proxy_config) -> str:
 
         return server
 
+    # scheme://user:pass@ in any URL inside a message
+    _URL_USERINFO = re.compile(r"([A-Za-z][A-Za-z0-9+.-]*://)[^/\s@]+@")
+
+    @classmethod
+    def _redact_proxy_credentials(cls, message: str, proxy_url: Optional[str]) -> str:
+        """Mask proxy credentials in an error message before it is raised.
+
+        Some aiohttp errors quote the proxy URL (``InvalidURL`` carries all of
+        it), and the errors raised from ``_handle_http`` end up in logs and in
+        crawl results.
+        """
+        if proxy_url and "@" in proxy_url:
+            from urllib.parse import unquote
+
+            userinfo = proxy_url.split("://", 1)[-1].rsplit("@", 1)[0]
+            # The exact string first: a password with "/" or "#" in it is not
+            # matched whole by the pattern below.
+            for secret in {userinfo, unquote(userinfo)}:
+                message = message.replace(secret + "@", "***@")
+        return cls._URL_USERINFO.sub(r"\1***@", message)
+
+    @classmethod
+    def _wrap_error(cls, exc_type, prefix: str, error: BaseException, proxy_url: Optional[str]):
+        """``exc_type(f"{prefix}: {error}")`` with proxy credentials masked.
+
+        When the original message carried credentials, it is also kept out of
+        the traceback, which would otherwise print it under "During handling
+        of the above exception".
+        """
+        text = str(error)
+        safe = cls._redact_proxy_credentials(text, proxy_url)
+        wrapped = exc_type(f"{prefix}: {safe}")
+        if safe != text:
+            wrapped.__suppress_context__ = True
+        return wrapped
+
     # Content types treated as text (decoded into html field)
     _TEXT_CONTENT_TYPES: Final = frozenset({
         'text/csv', 'text/plain', 'text/tab-separated-values', 'text/xml',
@@ -2807,23 +2844,23 @@ async def _handle_http(
 
             except aiohttp.ServerTimeoutError as e:
                 await self.hooks['on_error'](e)
-                raise ConnectionTimeoutError(f"Request timed out: {str(e)}")
+                raise self._wrap_error(ConnectionTimeoutError, "Request timed out", e, proxy_url)
 
             except aiohttp.ClientConnectorError as e:
                 await self.hooks['on_error'](e)
-                raise ConnectionError(f"Connection failed: {str(e)}")
+                raise self._wrap_error(ConnectionError, "Connection failed", e, proxy_url)
 
             except aiohttp.ClientError as e:
                 await self.hooks['on_error'](e)
-                raise HTTPCrawlerError(f"HTTP client error: {str(e)}")
+                raise self._wrap_error(HTTPCrawlerError, "HTTP client error", e, proxy_url)
 
             except asyncio.exceptions.TimeoutError as e:
                 await self.hooks['on_error'](e)
-                raise ConnectionTimeoutError(f"Request timed out: {str(e)}")
+                raise self._wrap_error(ConnectionTimeoutError, "Request timed out", e, proxy_url)
 
             except Exception as e:
                 await self.hooks['on_error'](e)
-                raise HTTPCrawlerError(f"HTTP request failed: {str(e)}")
+                raise self._wrap_error(HTTPCrawlerError, "HTTP request failed", e, proxy_url)
 
     async def crawl(
         self, 

tests/test_http_proxy_error_redaction_2326.py

@@ -0,0 +1,93 @@
+"""
+Tests for #2326: errors raised by AsyncHTTPCrawlerStrategy._handle_http must not
+carry the proxy password, in the message or in the traceback.
+
+Offline: the proxy is 127.0.0.1 on a port nothing listens on.
+"""
+import socket
+import traceback
+
+import pytest
+
+from crawl4ai.async_configs import CrawlerRunConfig, ProxyConfig
+from crawl4ai.async_crawler_strategy import (
+    AsyncHTTPCrawlerStrategy,
+    ConnectionTimeoutError,
+    HTTPCrawlerError,
+)
+
+redact = AsyncHTTPCrawlerStrategy._redact_proxy_credentials
+wrap = AsyncHTTPCrawlerStrategy._wrap_error
+
+
+def closed_port() -> int:
+    with socket.socket() as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+class TestRedactProxyCredentials:
+    @pytest.mark.parametrize("password", ["s3cret", "s3cret#pw", "pa/ss", "a?b", "50%off", "p@ss"])
+    def test_the_password_does_not_survive(self, password):
+        proxy_url = f"http://user:{password}@127.0.0.1:9"
+        message = f"Cannot connect via {proxy_url}: refused"
+        out = redact(message, proxy_url)
+        assert password not in out
+        assert "127.0.0.1:9" in out
+
+    def test_a_percent_encoded_proxy_url_is_matched_in_its_decoded_form(self):
+        proxy_url = "http://user:s3cret%23pw@127.0.0.1:9"
+        out = redact("http://user:s3cret#pw@127.0.0.1:9", proxy_url)
+        assert "s3cret" not in out
+
+    def test_credentials_in_any_url_are_masked_without_a_proxy_url(self):
+        out = redact("Proxy http://user:secret@host:8080 failed", None)
+        assert out == "Proxy http://***@host:8080 failed"
+
+    def test_a_message_without_credentials_is_unchanged(self):
+        message = "Cannot connect to host 127.0.0.1:9 ssl:default"
+        assert redact(message, "http://127.0.0.1:9") == message
+        assert redact(message, "http://user:secret@127.0.0.1:9") == message
+
+
+class TestWrapError:
+    def test_the_original_is_kept_out_of_the_traceback_when_it_held_credentials(self):
+        proxy_url = "http://user:s3cret#pw@127.0.0.1:9"
+        try:
+            try:
+                raise ValueError(proxy_url)
+            except ValueError as e:
+                raise wrap(HTTPCrawlerError, "HTTP client error", e, proxy_url)
+        except HTTPCrawlerError as err:
+            printed = "".join(traceback.format_exception(err))
+        assert "s3cret" not in printed
+        assert "HTTP client error: http://***@127.0.0.1:9" in printed
+
+    def test_the_original_stays_in_the_traceback_when_it_held_none(self):
+        try:
+            try:
+                raise TimeoutError("read timed out")
+            except TimeoutError as e:
+                raise wrap(ConnectionTimeoutError, "Request timed out", e, None)
+        except ConnectionTimeoutError as err:
+            printed = "".join(traceback.format_exception(err))
+            message = str(err)
+        assert "During handling of the above exception" in printed
+        assert message == "Request timed out: read timed out"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("password", ["s3cret", "s3cret#pw", "pa/ss"])
+async def test_handle_http_raises_without_the_password(password):
+    port = closed_port()
+    config = CrawlerRunConfig(
+        proxy_config=ProxyConfig(
+            server=f"http://127.0.0.1:{port}", username="user", password=password
+        )
+    )
+    strategy = AsyncHTTPCrawlerStrategy()
+    with pytest.raises(Exception) as caught:
+        await strategy._handle_http(f"http://127.0.0.1:{port}/", config)
+    await strategy.close()
+    printed = "".join(traceback.format_exception(caught.value))
+    assert password not in printed