From 2de19c8b0a16633ace19b4f2d292db06a276d4d5 Mon Sep 17 00:00:00 2001 From: omarbounawarapy Date: Thu, 1 Oct 2026 20:48:16 +0100 Subject: [PATCH 1/3] Percent-encode proxy credentials in the HTTP crawler proxy URL --- crawl4ai/async_crawler_strategy.py | 4 +++- tests/proxy/test_http_proxy_url_encoding.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tests/proxy/test_http_proxy_url_encoding.py diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py index 6d0fb4769..e46e50689 100644 --- a/crawl4ai/async_crawler_strategy.py +++ b/crawl4ai/async_crawler_strategy.py @@ -28,7 +28,7 @@ import aiohttp import chardet from aiohttp.client import ClientTimeout -from urllib.parse import urlparse +from urllib.parse import quote, urlparse from types import MappingProxyType import contextlib from functools import partial @@ -2642,6 +2642,8 @@ def _format_proxy_url(self, proxy_config) -> str: if username and password: # Insert credentials into URL: http://user:pass@host:port + username = quote(username, safe='') + password = quote(password, safe='') if '://' in server: protocol, rest = server.split('://', 1) return f"{protocol}://{username}:{password}@{rest}" diff --git a/tests/proxy/test_http_proxy_url_encoding.py b/tests/proxy/test_http_proxy_url_encoding.py new file mode 100644 index 000000000..f4bcbbd5f --- /dev/null +++ b/tests/proxy/test_http_proxy_url_encoding.py @@ -0,0 +1,16 @@ +import pytest +from yarl import URL + +from crawl4ai.async_configs import ProxyConfig +from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy + + +@pytest.mark.parametrize("password", ["plainpass", "p@ss", "p#ss", "pa/ss", "p?ss", "p%41ss"]) +def test_proxy_url_roundtrips_credentials(password): + strategy = AsyncHTTPCrawlerStrategy() + config = ProxyConfig(server="http://host:8080", username="us:er", password=password) + url = URL(strategy._format_proxy_url(config)) + assert url.host == "host" + assert url.port == 8080 + assert url.user == "us:er" + assert url.password == password From 2432985ddd2ec390d042a90390e1dae381bc88fd Mon Sep 17 00:00:00 2001 From: omarbounawarapy Date: Thu, 1 Oct 2026 21:08:18 +0100 Subject: [PATCH 2/3] Format new proxy URL test with black --- tests/proxy/test_http_proxy_url_encoding.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/proxy/test_http_proxy_url_encoding.py b/tests/proxy/test_http_proxy_url_encoding.py index f4bcbbd5f..e6d75d976 100644 --- a/tests/proxy/test_http_proxy_url_encoding.py +++ b/tests/proxy/test_http_proxy_url_encoding.py @@ -5,7 +5,9 @@ from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy -@pytest.mark.parametrize("password", ["plainpass", "p@ss", "p#ss", "pa/ss", "p?ss", "p%41ss"]) +@pytest.mark.parametrize( + "password", ["plainpass", "p@ss", "p#ss", "pa/ss", "p?ss", "p%41ss"] +) def test_proxy_url_roundtrips_credentials(password): strategy = AsyncHTTPCrawlerStrategy() config = ProxyConfig(server="http://host:8080", username="us:er", password=password) From 8b3a75da5fbc2a42fbe57565adb75287ed4db4a4 Mon Sep 17 00:00:00 2001 From: omarbounawarapy Date: Mon, 12 Oct 2026 13:46:02 +0100 Subject: [PATCH 3/3] Decode percent-encoded credentials in ProxyConfig.from_string --- crawl4ai/async_configs.py | 5 +++-- tests/proxy/test_http_proxy_url_encoding.py | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index e76cae18a..cc1efdb8e 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -3,6 +3,7 @@ import importlib import os import warnings +from urllib.parse import unquote import requests from .config import ( DEFAULT_PROVIDER, @@ -711,8 +712,8 @@ def from_string(proxy_str: str) -> "ProxyConfig": username, password = credentials.split(":", 1) return ProxyConfig( server=f"{protocol}://{server_part}", - username=username, - password=password, + username=unquote(username), + password=unquote(password), ) # URL without credentials (keep scheme) if "://" in s and "@" not in s: diff --git a/tests/proxy/test_http_proxy_url_encoding.py b/tests/proxy/test_http_proxy_url_encoding.py index e6d75d976..9e7515a13 100644 --- a/tests/proxy/test_http_proxy_url_encoding.py +++ b/tests/proxy/test_http_proxy_url_encoding.py @@ -16,3 +16,12 @@ def test_proxy_url_roundtrips_credentials(password): assert url.port == 8080 assert url.user == "us:er" assert url.password == password + + +def test_encoded_proxy_string_is_sent_once_encoded(): + strategy = AsyncHTTPCrawlerStrategy() + config = ProxyConfig.from_string("http://user:p%40ss@host:8080") + assert config.password == "p@ss" + url = URL(strategy._format_proxy_url(config)) + assert url.user == "user" + assert url.password == "p@ss"