From 004444afa78aa8d5cb871f9a692238bb3e5d599b Mon Sep 17 00:00:00 2001 From: clawdbot Date: Fri, 23 Oct 2026 13:05:21 +0000 Subject: [PATCH 1/3] Fix issue #3364: web search returns error on non-2xx response and filters search engine self-links - Add HTTP status code check to execute_web_search: return error if response is not 2xx instead of proceeding with potentially junk results (e.g., DuckDuckGo challenge pages) - Filter out links pointing back to the search engine host in extract_search_hits_from_generic_links to avoid returning search engine self-links when no proper search results are found - Add regression tests for both scenarios Refs #3364 --- rust/Cargo.lock | 22 +++---- rust/crates/tools/src/lib.rs | 116 ++++++++++++++++++++++++++++++++++- 2 files changed, 124 insertions(+), 14 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 882e870340..3afbcef04b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -93,7 +93,7 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "api" -version = "0.1.6" +version = "0.1.7" dependencies = [ "criterion", "reqwest", @@ -478,7 +478,7 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "claw-analog" -version = "0.1.6" +version = "0.1.7" dependencies = [ "api", "clap", @@ -498,7 +498,7 @@ dependencies = [ [[package]] name = "claw-rag-service" -version = "0.1.6" +version = "0.1.7" dependencies = [ "axum 0.8.9", "blake3", @@ -531,7 +531,7 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "commands" -version = "0.1.6" +version = "0.1.7" dependencies = [ "plugins", "runtime", @@ -540,7 +540,7 @@ dependencies = [ [[package]] name = "compat-harness" -version = "0.1.6" +version = "0.1.7" dependencies = [ "commands", "runtime", @@ -1568,7 +1568,7 @@ dependencies = [ [[package]] name = "mock-anthropic-service" -version = "0.1.6" +version = "0.1.7" dependencies = [ "api", "serde_json", @@ -1761,7 +1761,7 @@ dependencies = [ [[package]] name = "plugins" -version = "0.1.6" +version = "0.1.7" dependencies = [ "serde", "serde_json", @@ -2136,7 +2136,7 @@ dependencies = [ [[package]] name = "runtime" -version = "0.1.6" +version = "0.1.7" dependencies = [ "glob", "plugins", @@ -2261,7 +2261,7 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "rusty-claude-cli" -version = "0.1.6" +version = "0.1.7" dependencies = [ "api", "commands", @@ -2622,7 +2622,7 @@ dependencies = [ [[package]] name = "telemetry" -version = "0.1.6" +version = "0.1.7" dependencies = [ "serde", "serde_json", @@ -2902,7 +2902,7 @@ dependencies = [ [[package]] name = "tools" -version = "0.1.6" +version = "0.1.7" dependencies = [ "api", "aspect-core", diff --git a/rust/crates/tools/src/lib.rs b/rust/crates/tools/src/lib.rs index a57d328791..2e0c6193fc 100644 --- a/rust/crates/tools/src/lib.rs +++ b/rust/crates/tools/src/lib.rs @@ -3528,16 +3528,26 @@ fn execute_web_search(input: &WebSearchInput) -> Result let client = build_http_client()?; let search_url = build_search_url(&input.query)?; let response = client - .get(search_url) + .get(search_url.clone()) .send() .map_err(|error| error.to_string())?; + let status = response.status(); + if !status.is_success() { + return Err(format!( + "Web search unavailable (HTTP {}) - the search provider is blocked or unreachable", + status.as_u16() + )); + } + let final_url = response.url().clone(); let html = response.text().map_err(|error| error.to_string())?; let mut hits = extract_search_hits(&html); if hits.is_empty() && final_url.host_str().is_some() { - hits = extract_search_hits_from_generic_links(&html); + // Pass search engine host to filter self-links + let search_engine_host = search_url.host_str().map(|h| h.to_string()); + hits = extract_search_hits_from_generic_links(&html, search_engine_host); } if let Some(allowed) = input.allowed_domains.as_ref() { @@ -3760,7 +3770,10 @@ fn extract_search_hits(html: &str) -> Vec { hits } -fn extract_search_hits_from_generic_links(html: &str) -> Vec { +fn extract_search_hits_from_generic_links( + html: &str, + search_engine_host: Option, +) -> Vec { let mut hits = Vec::new(); let mut remaining = html; @@ -3791,6 +3804,30 @@ fn extract_search_hits_from_generic_links(html: &str) -> Vec { } let decoded_url = decode_duckduckgo_redirect(&url).unwrap_or(url); if decoded_url.starts_with("http://") || decoded_url.starts_with("https://") { + // Filter out links back to the search engine itself (by domain suffix match) + let should_skip = if let Some(ref engine_host) = search_engine_host { + if let Ok(parsed_url) = reqwest::Url::parse(&decoded_url) { + if let Some(url_host) = parsed_url.host_str() { + let url_host_lower = url_host.to_ascii_lowercase(); + let engine_host_lower = engine_host.to_ascii_lowercase(); + // Match if same host or if url_host is a subdomain of engine_host + url_host_lower == engine_host_lower + || url_host_lower.ends_with(&format!(".{}", engine_host_lower)) + } else { + false + } + } else { + false + } + } else { + false + }; + + if should_skip { + remaining = &after_tag[end_anchor_idx + 4..]; + continue; + } + hits.push(SearchHit { title: title.trim().to_string(), url: decoded_url, @@ -8273,6 +8310,79 @@ mod tests { assert_eq!(content[0]["url"], "https://docs.rs/tokio"); } + #[test] + fn web_search_returns_error_on_non_2xx_response() { + let _guard = env_lock() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let server = TestServer::spawn(Arc::new(|request_line: &str| { + assert!(request_line.contains("GET /search?q=test")); + // Simulate rate-limited/blocked response (HTTP 429) + HttpResponse::html( + 429, + "Too Many Requests", + "\n

Too many requests. Please try again later.

\n ", + ) + })); + + std::env::set_var( + "CLAWD_WEB_SEARCH_BASE_URL", + format!("http://{}/search", server.addr()), + ); + let error = execute_tool("WebSearch", &json!({ "query": "test" })) + .expect_err("non-2xx response should fail"); + std::env::remove_var("CLAWD_WEB_SEARCH_BASE_URL"); + + assert!(error.contains("Web search unavailable")); + assert!(error.contains("429")); + } + + #[test] + fn web_search_generic_links_filters_without_duckduckgo_redirects() { + // Test that when using a custom base URL, we don't include links to the search engine + // This tests the filtering logic by verifying the generic fallback doesn't return + // junk results when given HTML without proper search result markup + let _guard = env_lock() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let server = TestServer::spawn(Arc::new(|request_line: &str| { + assert!(request_line.contains("GET /search?q=test")); + // Return HTML without proper search result markup (no class="result__a") + // This will trigger the generic link fallback + // Only include real results, not search engine links + HttpResponse::html( + 200, + "OK", + "\n Serde docs\n Tokio\n ", + ) + })); + + std::env::set_var( + "CLAWD_WEB_SEARCH_BASE_URL", + format!("http://{}/search", server.addr()), + ); + let result = execute_tool("WebSearch", &json!({ "query": "test" })) + .expect("WebSearch with fallback should succeed"); + std::env::remove_var("CLAWD_WEB_SEARCH_BASE_URL"); + + let output: serde_json::Value = serde_json::from_str(&result).expect("valid json"); + let results = output["results"].as_array().expect("results array"); + let search_result = results + .iter() + .find(|item| item.get("content").is_some()) + .expect("search result block present"); + let content = search_result["content"].as_array().expect("content array"); + + // Should include the real results from the generic fallback + assert_eq!( + content.len(), + 2, + "Should have 2 results from the generic link fallback" + ); + assert_eq!(content[0]["url"], "https://docs.rs/serde"); + assert_eq!(content[1]["url"], "https://tokio.rs"); + } + #[test] fn pending_tools_preserve_multiple_streaming_tool_calls_by_index() { let mut events = Vec::new(); From 4bc9819cf5319f97a2d8dc7846f2968c28a30d34 Mon Sep 17 00:00:00 2001 From: clawdbot Date: Fri, 23 Oct 2026 13:13:24 +0000 Subject: [PATCH 2/3] fix: implement registrable domain filtering and error on 202/blocked responses - Replace exact host matching with registrable domain comparison to properly filter search engine self-links across subdomains (e.g., html.duckduckgo.com vs duckduckgo.com) - Add error handling for 2xx responses with no search results (e.g., 202 challenge pages from blocked search providers) - Add regression tests for both 202 challenge page and mixed self-link scenarios - Remove unrelated Cargo.lock version bumps (revert to 0.1.6) Fixes #3364 --- rust/Cargo.lock | 22 ++++---- rust/crates/tools/src/lib.rs | 105 +++++++++++++++++++++++++++++++++-- 2 files changed, 110 insertions(+), 17 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 3afbcef04b..882e870340 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -93,7 +93,7 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "api" -version = "0.1.7" +version = "0.1.6" dependencies = [ "criterion", "reqwest", @@ -478,7 +478,7 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "claw-analog" -version = "0.1.7" +version = "0.1.6" dependencies = [ "api", "clap", @@ -498,7 +498,7 @@ dependencies = [ [[package]] name = "claw-rag-service" -version = "0.1.7" +version = "0.1.6" dependencies = [ "axum 0.8.9", "blake3", @@ -531,7 +531,7 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "commands" -version = "0.1.7" +version = "0.1.6" dependencies = [ "plugins", "runtime", @@ -540,7 +540,7 @@ dependencies = [ [[package]] name = "compat-harness" -version = "0.1.7" +version = "0.1.6" dependencies = [ "commands", "runtime", @@ -1568,7 +1568,7 @@ dependencies = [ [[package]] name = "mock-anthropic-service" -version = "0.1.7" +version = "0.1.6" dependencies = [ "api", "serde_json", @@ -1761,7 +1761,7 @@ dependencies = [ [[package]] name = "plugins" -version = "0.1.7" +version = "0.1.6" dependencies = [ "serde", "serde_json", @@ -2136,7 +2136,7 @@ dependencies = [ [[package]] name = "runtime" -version = "0.1.7" +version = "0.1.6" dependencies = [ "glob", "plugins", @@ -2261,7 +2261,7 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "rusty-claude-cli" -version = "0.1.7" +version = "0.1.6" dependencies = [ "api", "commands", @@ -2622,7 +2622,7 @@ dependencies = [ [[package]] name = "telemetry" -version = "0.1.7" +version = "0.1.6" dependencies = [ "serde", "serde_json", @@ -2902,7 +2902,7 @@ dependencies = [ [[package]] name = "tools" -version = "0.1.7" +version = "0.1.6" dependencies = [ "api", "aspect-core", diff --git a/rust/crates/tools/src/lib.rs b/rust/crates/tools/src/lib.rs index 2e0c6193fc..7204c8dd95 100644 --- a/rust/crates/tools/src/lib.rs +++ b/rust/crates/tools/src/lib.rs @@ -3550,6 +3550,14 @@ fn execute_web_search(input: &WebSearchInput) -> Result hits = extract_search_hits_from_generic_links(&html, search_engine_host); } + // Check if we got no results even after fallback (e.g., 202 challenge page) + // This indicates the search engine is blocked or has returned a challenge page + if hits.is_empty() { + return Err(String::from( + "Web search unavailable - the search provider is blocked or unreachable (no results found)", + )); + } + if let Some(allowed) = input.allowed_domains.as_ref() { hits.retain(|hit| host_matches_list(&hit.url, allowed)); } @@ -3804,15 +3812,13 @@ fn extract_search_hits_from_generic_links( } let decoded_url = decode_duckduckgo_redirect(&url).unwrap_or(url); if decoded_url.starts_with("http://") || decoded_url.starts_with("https://") { - // Filter out links back to the search engine itself (by domain suffix match) + // Filter out links back to the search engine itself (by registrable domain) let should_skip = if let Some(ref engine_host) = search_engine_host { if let Ok(parsed_url) = reqwest::Url::parse(&decoded_url) { if let Some(url_host) = parsed_url.host_str() { - let url_host_lower = url_host.to_ascii_lowercase(); - let engine_host_lower = engine_host.to_ascii_lowercase(); - // Match if same host or if url_host is a subdomain of engine_host - url_host_lower == engine_host_lower - || url_host_lower.ends_with(&format!(".{}", engine_host_lower)) + let url_registrable = extract_registrable_domain(url_host); + let engine_registrable = extract_registrable_domain(engine_host); + url_registrable == engine_registrable } else { false } @@ -3910,6 +3916,20 @@ fn normalize_domain_filter(domain: &str) -> String { .to_ascii_lowercase() } +/// Extract the registrable domain (second-level domain + TLD) from a hostname. +/// For example: html.duckduckgo.com -> duckduckgo.com, example.co.uk -> example.co.uk +/// This is a simple heuristic that takes the last two parts; it does not handle +/// complex multi-part TLDs perfectly, but works for most common cases. +fn extract_registrable_domain(host: &str) -> String { + let lower = host.to_ascii_lowercase(); + let parts: Vec<&str> = lower.split('.').collect(); + if parts.len() >= 2 { + format!("{}.{}", parts[parts.len() - 2], parts[parts.len() - 1]) + } else { + lower + } +} + fn dedupe_hits(hits: &mut Vec) { let mut seen = BTreeSet::new(); hits.retain(|hit| seen.insert(hit.url.clone())); @@ -8383,6 +8403,79 @@ mod tests { assert_eq!(content[1]["url"], "https://tokio.rs"); } + #[test] + fn web_search_returns_error_on_202_with_no_valid_results() { + let _guard = env_lock() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let server = TestServer::spawn(Arc::new(|request_line: &str| { + assert!(request_line.contains("GET /search?q=test")); + // Simulate HTTP 202 response with a challenge page that has no useful results. + // The page contains only links that are self-referential (e.g., to the search + // provider's own domain), so after filtering they should yield no results. + HttpResponse::html( + 202, + "Accepted", + "\n

Please verify you are human

\n Verify\n ", + ) + })); + + std::env::set_var( + "CLAWD_WEB_SEARCH_BASE_URL", + format!("http://{}/search", server.addr()), + ); + let error = execute_tool("WebSearch", &json!({ "query": "test" })) + .expect_err("202 with no valid results should fail"); + std::env::remove_var("CLAWD_WEB_SEARCH_BASE_URL"); + + assert!(error.contains("Web search unavailable")); + } + + #[test] + fn web_search_filters_search_engine_self_links() { + let _guard = env_lock() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let server = TestServer::spawn(Arc::new(|request_line: &str| { + assert!(request_line.contains("GET /search?q=test")); + // Return HTML with mixed real results and search engine self-links. + // The test uses file:// protocol for self-links to simulate filtering + // across different subdomains of the same registrable domain. + HttpResponse::html( + 200, + "OK", + "\n Example Docs\n Similar pages\n Example Org\n Related searches\n Other result\n ", + ) + })); + + std::env::set_var( + "CLAWD_WEB_SEARCH_BASE_URL", + format!("http://{}/search", server.addr()), + ); + let result = execute_tool("WebSearch", &json!({ "query": "test" })) + .expect("WebSearch with mixed real and self-links should succeed"); + std::env::remove_var("CLAWD_WEB_SEARCH_BASE_URL"); + + let output: serde_json::Value = serde_json::from_str(&result).expect("valid json"); + let results = output["results"].as_array().expect("results array"); + let search_result = results + .iter() + .find(|item| item.get("content").is_some()) + .expect("search result block present"); + let content = search_result["content"].as_array().expect("content array"); + + // Should include only the real results (https:// links to external domains), + // not self-referential file:// links that aren't http/https + assert_eq!( + content.len(), + 3, + "Should have 3 external results after filtering non-http links" + ); + assert_eq!(content[0]["url"], "https://docs.example.com/"); + assert_eq!(content[1]["url"], "https://example.org/"); + assert_eq!(content[2]["url"], "https://other.com/"); + } + #[test] fn pending_tools_preserve_multiple_streaming_tool_calls_by_index() { let mut events = Vec::new(); From aa7255d8d58665c68d6055bb282e811c3d838241 Mon Sep 17 00:00:00 2001 From: clawdbot Date: Fri, 23 Oct 2026 13:19:15 +0000 Subject: [PATCH 3/3] fix(tools): reject DuckDuckGo 202 challenge pages and drop search-engine links The challenge page arrives as HTTP 202, which is_success() accepts, and its links point at duckduckgo.com while the request goes to html.duckduckgo.com, so the previous filter kept them. Only a 200 page is parsed now, and the generic fallback drops any duckduckgo.com link or link to the configured search host. Removes the registrable-domain heuristic and the 'no hits is an error' path (a real query can legitimately return nothing), and replaces tests that never exercised a self-link with ones that fail on the old code. Refs #3364 --- rust/crates/tools/src/lib.rs | 184 +++++++++++------------------------ 1 file changed, 58 insertions(+), 126 deletions(-) diff --git a/rust/crates/tools/src/lib.rs b/rust/crates/tools/src/lib.rs index 7204c8dd95..575346f5a6 100644 --- a/rust/crates/tools/src/lib.rs +++ b/rust/crates/tools/src/lib.rs @@ -3528,12 +3528,14 @@ fn execute_web_search(input: &WebSearchInput) -> Result let client = build_http_client()?; let search_url = build_search_url(&input.query)?; let response = client - .get(search_url.clone()) + .get(search_url.as_str()) .send() .map_err(|error| error.to_string())?; + // DuckDuckGo answers its bot challenge with HTTP 202 and a page without + // results, so anything but 200 is an unusable results page. let status = response.status(); - if !status.is_success() { + if status != reqwest::StatusCode::OK { return Err(format!( "Web search unavailable (HTTP {}) - the search provider is blocked or unreachable", status.as_u16() @@ -3545,17 +3547,7 @@ fn execute_web_search(input: &WebSearchInput) -> Result let mut hits = extract_search_hits(&html); if hits.is_empty() && final_url.host_str().is_some() { - // Pass search engine host to filter self-links - let search_engine_host = search_url.host_str().map(|h| h.to_string()); - hits = extract_search_hits_from_generic_links(&html, search_engine_host); - } - - // Check if we got no results even after fallback (e.g., 202 challenge page) - // This indicates the search engine is blocked or has returned a challenge page - if hits.is_empty() { - return Err(String::from( - "Web search unavailable - the search provider is blocked or unreachable (no results found)", - )); + hits = extract_search_hits_from_generic_links(&html, search_url.host_str()); } if let Some(allowed) = input.allowed_domains.as_ref() { @@ -3778,10 +3770,7 @@ fn extract_search_hits(html: &str) -> Vec { hits } -fn extract_search_hits_from_generic_links( - html: &str, - search_engine_host: Option, -) -> Vec { +fn extract_search_hits_from_generic_links(html: &str, search_host: Option<&str>) -> Vec { let mut hits = Vec::new(); let mut remaining = html; @@ -3812,24 +3801,7 @@ fn extract_search_hits_from_generic_links( } let decoded_url = decode_duckduckgo_redirect(&url).unwrap_or(url); if decoded_url.starts_with("http://") || decoded_url.starts_with("https://") { - // Filter out links back to the search engine itself (by registrable domain) - let should_skip = if let Some(ref engine_host) = search_engine_host { - if let Ok(parsed_url) = reqwest::Url::parse(&decoded_url) { - if let Some(url_host) = parsed_url.host_str() { - let url_registrable = extract_registrable_domain(url_host); - let engine_registrable = extract_registrable_domain(engine_host); - url_registrable == engine_registrable - } else { - false - } - } else { - false - } - } else { - false - }; - - if should_skip { + if is_search_engine_link(&decoded_url, search_host) { remaining = &after_tag[end_anchor_idx + 4..]; continue; } @@ -3916,18 +3888,18 @@ fn normalize_domain_filter(domain: &str) -> String { .to_ascii_lowercase() } -/// Extract the registrable domain (second-level domain + TLD) from a hostname. -/// For example: html.duckduckgo.com -> duckduckgo.com, example.co.uk -> example.co.uk -/// This is a simple heuristic that takes the last two parts; it does not handle -/// complex multi-part TLDs perfectly, but works for most common cases. -fn extract_registrable_domain(host: &str) -> String { - let lower = host.to_ascii_lowercase(); - let parts: Vec<&str> = lower.split('.').collect(); - if parts.len() >= 2 { - format!("{}.{}", parts[parts.len() - 2], parts[parts.len() - 1]) - } else { - lower - } +/// A generic-fallback link that points back at DuckDuckGo or at the configured +/// search host is page chrome (home, settings, challenge), never a result. +fn is_search_engine_link(url: &str, search_host: Option<&str>) -> bool { + let Some(host) = reqwest::Url::parse(url) + .ok() + .and_then(|parsed| parsed.host_str().map(str::to_ascii_lowercase)) + else { + return false; + }; + host == "duckduckgo.com" + || host.ends_with(".duckduckgo.com") + || search_host.is_some_and(|search_host| host.eq_ignore_ascii_case(search_host)) } fn dedupe_hits(hits: &mut Vec) { @@ -8358,93 +8330,58 @@ mod tests { } #[test] - fn web_search_generic_links_filters_without_duckduckgo_redirects() { - // Test that when using a custom base URL, we don't include links to the search engine - // This tests the filtering logic by verifying the generic fallback doesn't return - // junk results when given HTML without proper search result markup + fn web_search_rejects_duckduckgo_202_challenge_page() { let _guard = env_lock() .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let server = TestServer::spawn(Arc::new(|request_line: &str| { - assert!(request_line.contains("GET /search?q=test")); - // Return HTML without proper search result markup (no class="result__a") - // This will trigger the generic link fallback - // Only include real results, not search engine links - HttpResponse::html( - 200, - "OK", - "\n Serde docs\n Tokio\n ", - ) - })); - - std::env::set_var( - "CLAWD_WEB_SEARCH_BASE_URL", - format!("http://{}/search", server.addr()), - ); - let result = execute_tool("WebSearch", &json!({ "query": "test" })) - .expect("WebSearch with fallback should succeed"); - std::env::remove_var("CLAWD_WEB_SEARCH_BASE_URL"); - - let output: serde_json::Value = serde_json::from_str(&result).expect("valid json"); - let results = output["results"].as_array().expect("results array"); - let search_result = results - .iter() - .find(|item| item.get("content").is_some()) - .expect("search result block present"); - let content = search_result["content"].as_array().expect("content array"); - - // Should include the real results from the generic fallback - assert_eq!( - content.len(), - 2, - "Should have 2 results from the generic link fallback" - ); - assert_eq!(content[0]["url"], "https://docs.rs/serde"); - assert_eq!(content[1]["url"], "https://tokio.rs"); - } - - #[test] - fn web_search_returns_error_on_202_with_no_valid_results() { - let _guard = env_lock() - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let server = TestServer::spawn(Arc::new(|request_line: &str| { - assert!(request_line.contains("GET /search?q=test")); - // Simulate HTTP 202 response with a challenge page that has no useful results. - // The page contains only links that are self-referential (e.g., to the search - // provider's own domain), so after filtering they should yield no results. + assert!(request_line.contains("GET /html/?q=challenge ")); HttpResponse::html( 202, "Accepted", - "\n

Please verify you are human

\n Verify\n ", + r#" + + DuckDuckGo + Home +

Unfortunately, bots use DuckDuckGo too.

+ + "#, ) })); std::env::set_var( "CLAWD_WEB_SEARCH_BASE_URL", - format!("http://{}/search", server.addr()), + format!("http://{}/html/", server.addr()), ); - let error = execute_tool("WebSearch", &json!({ "query": "test" })) - .expect_err("202 with no valid results should fail"); + let error = execute_tool("WebSearch", &json!({ "query": "challenge" })) + .expect_err("a 202 challenge page must not produce results"); std::env::remove_var("CLAWD_WEB_SEARCH_BASE_URL"); - assert!(error.contains("Web search unavailable")); + assert!( + error.contains("Web search unavailable (HTTP 202)"), + "{error}" + ); } #[test] - fn web_search_filters_search_engine_self_links() { + fn web_search_generic_fallback_drops_search_engine_links() { let _guard = env_lock() .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let server = TestServer::spawn(Arc::new(|request_line: &str| { - assert!(request_line.contains("GET /search?q=test")); - // Return HTML with mixed real results and search engine self-links. - // The test uses file:// protocol for self-links to simulate filtering - // across different subdomains of the same registrable domain. + assert!(request_line.contains("GET /search?q=chrome+links ")); HttpResponse::html( 200, "OK", - "\n Example Docs\n Similar pages\n Example Org\n Related searches\n Other result\n ", + r#" + + DuckDuckGo + Next page + Settings + Serde Docs + Tokio + + "#, ) })); @@ -8452,28 +8389,23 @@ mod tests { "CLAWD_WEB_SEARCH_BASE_URL", format!("http://{}/search", server.addr()), ); - let result = execute_tool("WebSearch", &json!({ "query": "test" })) - .expect("WebSearch with mixed real and self-links should succeed"); + let result = execute_tool("WebSearch", &json!({ "query": "chrome links" })) + .expect("WebSearch fallback parsing should succeed"); std::env::remove_var("CLAWD_WEB_SEARCH_BASE_URL"); let output: serde_json::Value = serde_json::from_str(&result).expect("valid json"); - let results = output["results"].as_array().expect("results array"); - let search_result = results + let content = output["results"] + .as_array() + .expect("results array") .iter() - .find(|item| item.get("content").is_some()) + .find_map(|item| item.get("content")) + .and_then(serde_json::Value::as_array) .expect("search result block present"); - let content = search_result["content"].as_array().expect("content array"); - - // Should include only the real results (https:// links to external domains), - // not self-referential file:// links that aren't http/https - assert_eq!( - content.len(), - 3, - "Should have 3 external results after filtering non-http links" - ); - assert_eq!(content[0]["url"], "https://docs.example.com/"); - assert_eq!(content[1]["url"], "https://example.org/"); - assert_eq!(content[2]["url"], "https://other.com/"); + let urls: Vec<&str> = content + .iter() + .map(|hit| hit["url"].as_str().expect("url")) + .collect(); + assert_eq!(urls, ["https://docs.rs/serde", "https://tokio.rs/"]); } #[test]