From 018053b152ed046481139158bbd5f19f28df5eb1 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Tue, 13 Oct 2026 16:13:21 +0200 Subject: [PATCH 1/2] fix(cli): honor -c cache_mode instead of always bypassing the cache (#2365) --bypass-cache was a flag with default=True, so it always forced CacheMode.BYPASS after -c and config files were applied. Default it to False; CrawlerRunConfig already defaults to BYPASS, so plain crwl still skips the cache. -c and YAML/JSON configs also pass cache_mode as a string, which never matched the CacheMode enum, so CrawlerRunConfig now converts it. --- crawl4ai/async_configs.py | 3 ++- crawl4ai/cli.py | 4 ++-- tests/cli/test_cli.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index ad51cb178..fbea1e3f0 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -1942,7 +1942,8 @@ def __init__( self.fetch_ssl_certificate = fetch_ssl_certificate # Caching Parameters - self.cache_mode = cache_mode + # Accept "enabled" etc. from CLI -c and YAML/JSON config files + self.cache_mode = CacheMode(cache_mode.lower()) if isinstance(cache_mode, str) else cache_mode self.session_id = session_id self.bypass_cache = bypass_cache self.disable_cache = disable_cache diff --git a/crawl4ai/cli.py b/crawl4ai/cli.py index f82864a18..ecf333c23 100644 --- a/crawl4ai/cli.py +++ b/crawl4ai/cli.py @@ -1023,7 +1023,7 @@ def cdp_cmd(user_data_dir: Optional[str], port: int, browser_type: str, headless @click.option("--crawler", "-c", type=str, callback=parse_key_values, help="Crawler parameters as key1=value1,key2=value2") @click.option("--output", "-o", type=click.Choice(["all", "json", "markdown", "md", "markdown-fit", "md-fit"]), default="all", help="json = extracted data only (needs -j or -e). Use 'all' (default) for the full crawl result.") @click.option("--output-file", "-O", type=click.Path(), help="Output file path (default: stdout)") -@click.option("--bypass-cache", "-bc", is_flag=True, default=True, help="Bypass cache when crawling") +@click.option("--bypass-cache", "-bc", is_flag=True, default=False, help="Bypass cache when crawling (the default unless -c cache_mode=... is set)") @click.option("--question", "-q", help="Ask a question about the crawled content") @click.option("--verbose", "-v", is_flag=True) @click.option("--profile", "-p", help="Use a specific browser profile (by name)") @@ -1587,7 +1587,7 @@ def shrink_cmd(profile_name: str, level: str, dry_run: bool): @click.option("--browser", "-b", type=str, callback=parse_key_values, help="Browser parameters as key1=value1,key2=value2") @click.option("--crawler", "-c", type=str, callback=parse_key_values, help="Crawler parameters as key1=value1,key2=value2") @click.option("--output", "-o", type=click.Choice(["all", "json", "markdown", "md", "markdown-fit", "md-fit"]), default="all", help="json = extracted data only (needs -j or -e). Use 'all' (default) for the full crawl result.") -@click.option("--bypass-cache", is_flag=True, default=True, help="Bypass cache when crawling") +@click.option("--bypass-cache", is_flag=True, default=False, help="Bypass cache when crawling (the default unless -c cache_mode=... is set)") @click.option("--question", "-q", help="Ask a question about the crawled content") @click.option("--verbose", "-v", is_flag=True) @click.option("--profile", "-p", help="Use a specific browser profile (by name)") diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 1cf4b166c..67770cbdc 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -268,5 +268,23 @@ def test_json_output_with_question_is_allowed(runner): run.assert_called() # no early error +def _crawler_cfg(runner, *args): + with patch("crawl4ai.cli.anyio.run") as run: + runner.invoke(cli, ["crawl", "https://example.com", "-o", "md", *args]) + return run.call_args.args[3] + + +def test_cache_bypassed_by_default(runner): + from crawl4ai import CacheMode + assert _crawler_cfg(runner).cache_mode == CacheMode.BYPASS + + +def test_crawler_param_cache_mode_is_honored(runner): + # #2365: -c cache_mode=enabled was overridden and kept as a plain string + from crawl4ai import CacheMode + assert _crawler_cfg(runner, "-c", "cache_mode=enabled").cache_mode == CacheMode.ENABLED + assert _crawler_cfg(runner, "-c", "cache_mode=enabled", "-bc").cache_mode == CacheMode.BYPASS + + if __name__ == '__main__': pytest.main(['-v', '-s', '--tb=native', __file__]) From ae4a9b60a5c5a79fcba5e4fb3879440d12dfa544 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Wed, 14 Oct 2026 09:23:35 +0200 Subject: [PATCH 2/2] fix(api): refuse an invalid cache_mode string with 400, not 500 CrawlerRunConfig now converts a string cache_mode to CacheMode and raises ValueError on an unknown one, which the Docker API turned into a 500. Check it in the untrusted gate so /crawl and /crawl/stream answer 400. The playground template test compared the raw "bypass" string with the loaded config; cache_mode is now a CacheMode, so compare enum values. --- crawl4ai/async_configs.py | 4 ++++ .../tests/test_playground_config_protocol.py | 6 +++++- tests/unit/test_config_provenance.py | 14 ++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index fbea1e3f0..d499f6e99 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -427,6 +427,10 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict: raise UntrustedConfigError( "fetch_ssl_certificate cannot be combined with deep_crawl_strategy from an untrusted request" ) + cache_mode = out.get("cache_mode") if type_name == "CrawlerRunConfig" else None + if isinstance(cache_mode, str) and cache_mode.lower() not in {m.value for m in CacheMode}: + # CrawlerRunConfig raises ValueError on it, which would surface as a 500. + raise UntrustedConfigError(f"invalid cache_mode: {cache_mode!r}") return out diff --git a/deploy/docker/tests/test_playground_config_protocol.py b/deploy/docker/tests/test_playground_config_protocol.py index 926b113d4..d71529de1 100644 --- a/deploy/docker/tests/test_playground_config_protocol.py +++ b/deploy/docker/tests/test_playground_config_protocol.py @@ -90,7 +90,11 @@ def test_every_template_field_reaches_the_config_object(self): obj = CONFIG_CLASSES[name].load( {"type": name, "params": params}, provenance=Provenance.UNTRUSTED ) - applied = {key: getattr(obj, key) for key in params} + # Enum fields (cache_mode) come back as the enum; compare by value. + applied = { + key: getattr(getattr(obj, key), "value", getattr(obj, key)) + for key in params + } self.assertEqual(applied, params) def test_stream_reaches_the_shape_should_use_stream_reads(self): diff --git a/tests/unit/test_config_provenance.py b/tests/unit/test_config_provenance.py index 62f167a6f..201629c5a 100644 --- a/tests/unit/test_config_provenance.py +++ b/tests/unit/test_config_provenance.py @@ -195,3 +195,17 @@ def test_url_pattern_star_led_glob_is_linear_and_unchanged(): start = time.time() assert not f.apply("https://x.com/" + "a" * 50000) assert time.time() - start < 0.5 + + +def test_cache_mode_string_is_converted_and_invalid_one_refused(): + from crawl4ai import CacheMode + + cfg = CrawlerRunConfig.load( + {"type": "CrawlerRunConfig", "params": {"cache_mode": "Enabled"}}, provenance=UNTRUSTED + ) + assert cfg.cache_mode == CacheMode.ENABLED + # A bad value must be a 400 (UntrustedConfigError), not a ValueError (500). + with pytest.raises(UntrustedConfigError, match="cache_mode"): + CrawlerRunConfig.load( + {"type": "CrawlerRunConfig", "params": {"cache_mode": "nope"}}, provenance=UNTRUSTED + )