diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py index 6d0fb4769..e10a2fe19 100644 --- a/crawl4ai/async_crawler_strategy.py +++ b/crawl4ai/async_crawler_strategy.py @@ -13,6 +13,7 @@ from PIL import Image, ImageDraw, ImageFont import hashlib import random +import re import uuid from .js_snippet import load_js_script from .models import AsyncCrawlResponse @@ -2650,6 +2651,42 @@ def _format_proxy_url(self, proxy_config) -> str: return server + # scheme://user:pass@ in any URL inside a message + _URL_USERINFO = re.compile(r"([A-Za-z][A-Za-z0-9+.-]*://)[^/\s@]+@") + + @classmethod + def _redact_proxy_credentials(cls, message: str, proxy_url: Optional[str]) -> str: + """Mask proxy credentials in an error message before it is raised. + + Some aiohttp errors quote the proxy URL (``InvalidURL`` carries all of + it), and the errors raised from ``_handle_http`` end up in logs and in + crawl results. + """ + if proxy_url and "@" in proxy_url: + from urllib.parse import unquote + + userinfo = proxy_url.split("://", 1)[-1].rsplit("@", 1)[0] + # The exact string first: a password with "/" or "#" in it is not + # matched whole by the pattern below. + for secret in {userinfo, unquote(userinfo)}: + message = message.replace(secret + "@", "***@") + return cls._URL_USERINFO.sub(r"\1***@", message) + + @classmethod + def _wrap_error(cls, exc_type, prefix: str, error: BaseException, proxy_url: Optional[str]): + """``exc_type(f"{prefix}: {error}")`` with proxy credentials masked. + + When the original message carried credentials, it is also kept out of + the traceback, which would otherwise print it under "During handling + of the above exception". + """ + text = str(error) + safe = cls._redact_proxy_credentials(text, proxy_url) + wrapped = exc_type(f"{prefix}: {safe}") + if safe != text: + wrapped.__suppress_context__ = True + return wrapped + # Content types treated as text (decoded into html field) _TEXT_CONTENT_TYPES: Final = frozenset({ 'text/csv', 'text/plain', 'text/tab-separated-values', 'text/xml', @@ -2807,23 +2844,23 @@ async def _handle_http( except aiohttp.ServerTimeoutError as e: await self.hooks['on_error'](e) - raise ConnectionTimeoutError(f"Request timed out: {str(e)}") + raise self._wrap_error(ConnectionTimeoutError, "Request timed out", e, proxy_url) except aiohttp.ClientConnectorError as e: await self.hooks['on_error'](e) - raise ConnectionError(f"Connection failed: {str(e)}") + raise self._wrap_error(ConnectionError, "Connection failed", e, proxy_url) except aiohttp.ClientError as e: await self.hooks['on_error'](e) - raise HTTPCrawlerError(f"HTTP client error: {str(e)}") + raise self._wrap_error(HTTPCrawlerError, "HTTP client error", e, proxy_url) except asyncio.exceptions.TimeoutError as e: await self.hooks['on_error'](e) - raise ConnectionTimeoutError(f"Request timed out: {str(e)}") + raise self._wrap_error(ConnectionTimeoutError, "Request timed out", e, proxy_url) except Exception as e: await self.hooks['on_error'](e) - raise HTTPCrawlerError(f"HTTP request failed: {str(e)}") + raise self._wrap_error(HTTPCrawlerError, "HTTP request failed", e, proxy_url) async def crawl( self, diff --git a/tests/test_http_proxy_error_redaction_2326.py b/tests/test_http_proxy_error_redaction_2326.py new file mode 100644 index 000000000..d501ffebe --- /dev/null +++ b/tests/test_http_proxy_error_redaction_2326.py @@ -0,0 +1,93 @@ +""" +Tests for #2326: errors raised by AsyncHTTPCrawlerStrategy._handle_http must not +carry the proxy password, in the message or in the traceback. + +Offline: the proxy is 127.0.0.1 on a port nothing listens on. +""" +import socket +import traceback + +import pytest + +from crawl4ai.async_configs import CrawlerRunConfig, ProxyConfig +from crawl4ai.async_crawler_strategy import ( + AsyncHTTPCrawlerStrategy, + ConnectionTimeoutError, + HTTPCrawlerError, +) + +redact = AsyncHTTPCrawlerStrategy._redact_proxy_credentials +wrap = AsyncHTTPCrawlerStrategy._wrap_error + + +def closed_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +class TestRedactProxyCredentials: + @pytest.mark.parametrize("password", ["s3cret", "s3cret#pw", "pa/ss", "a?b", "50%off", "p@ss"]) + def test_the_password_does_not_survive(self, password): + proxy_url = f"http://user:{password}@127.0.0.1:9" + message = f"Cannot connect via {proxy_url}: refused" + out = redact(message, proxy_url) + assert password not in out + assert "127.0.0.1:9" in out + + def test_a_percent_encoded_proxy_url_is_matched_in_its_decoded_form(self): + proxy_url = "http://user:s3cret%23pw@127.0.0.1:9" + out = redact("http://user:s3cret#pw@127.0.0.1:9", proxy_url) + assert "s3cret" not in out + + def test_credentials_in_any_url_are_masked_without_a_proxy_url(self): + out = redact("Proxy http://user:secret@host:8080 failed", None) + assert out == "Proxy http://***@host:8080 failed" + + def test_a_message_without_credentials_is_unchanged(self): + message = "Cannot connect to host 127.0.0.1:9 ssl:default" + assert redact(message, "http://127.0.0.1:9") == message + assert redact(message, "http://user:secret@127.0.0.1:9") == message + + +class TestWrapError: + def test_the_original_is_kept_out_of_the_traceback_when_it_held_credentials(self): + proxy_url = "http://user:s3cret#pw@127.0.0.1:9" + try: + try: + raise ValueError(proxy_url) + except ValueError as e: + raise wrap(HTTPCrawlerError, "HTTP client error", e, proxy_url) + except HTTPCrawlerError as err: + printed = "".join(traceback.format_exception(err)) + assert "s3cret" not in printed + assert "HTTP client error: http://***@127.0.0.1:9" in printed + + def test_the_original_stays_in_the_traceback_when_it_held_none(self): + try: + try: + raise TimeoutError("read timed out") + except TimeoutError as e: + raise wrap(ConnectionTimeoutError, "Request timed out", e, None) + except ConnectionTimeoutError as err: + printed = "".join(traceback.format_exception(err)) + message = str(err) + assert "During handling of the above exception" in printed + assert message == "Request timed out: read timed out" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("password", ["s3cret", "s3cret#pw", "pa/ss"]) +async def test_handle_http_raises_without_the_password(password): + port = closed_port() + config = CrawlerRunConfig( + proxy_config=ProxyConfig( + server=f"http://127.0.0.1:{port}", username="user", password=password + ) + ) + strategy = AsyncHTTPCrawlerStrategy() + with pytest.raises(Exception) as caught: + await strategy._handle_http(f"http://127.0.0.1:{port}/", config) + await strategy.close() + printed = "".join(traceback.format_exception(caught.value)) + assert password not in printed