From 7a31f1b1fae6fe6a4191de9234e2b64dcc9de011 Mon Sep 17 00:00:00 2001 From: Josef Schlehofer Date: Sun, 27 Sep 2026 07:49:19 +0200 Subject: [PATCH] fix: fail the run when no selected server answers When all selected servers are down, or there is none to test, exit 1 and report each server's failure to stderr instead of emitting an empty report. Runs with at least one successful server still exit 0. As in the Go client, the probe reads at most 512 bytes of the ping response and names a wrong status with Go's reason phrase. The parity test now runs against the Go commit with the same fix. --- .github/workflows/ci.yml | 2 +- src/defs/server.rs | 75 ++++++- src/helper.rs | 47 ++++- tests/data/parity/servers-empty.json | 1 + tests/integration.rs | 284 +++++++++++++++++++++++++++ tests/parity/cases.rs | 24 ++- tests/parity/main.rs | 8 +- 7 files changed, 426 insertions(+), 15 deletions(-) create mode 100644 tests/data/parity/servers-empty.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29e18f8..5c96e8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,7 +113,7 @@ jobs: env: # The commit the expectations were recorded against. The test is given # it too and refuses to run against another one. - LIBRESPEED_GO_COMMIT: b660d1e6c24f14fc93624538d9e73163e7784335 + LIBRESPEED_GO_COMMIT: 8aefd77bd4bde428cd86d3622102f0ac0443d96e steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/src/defs/server.rs b/src/defs/server.rs index 9021763..4c668a0 100644 --- a/src/defs/server.rs +++ b/src/defs/server.rs @@ -127,7 +127,7 @@ impl Server { Ok(u) => url_join_path(&u, &self.ping_url), Err(e) => { write_debug!(out, "Failed when creating HTTP request: {e}\n"); - return ServerStatus::default(); + return ServerStatus::down(format!("{e:#}")); } }; @@ -164,16 +164,31 @@ impl Server { "Failed when parsing get IP result: {}\n", GoQuote(&body) ); - return ServerStatus::default(); } + // Status first: a web server's 404 or 500 comes with an error + // page, and the status names that fault better than the body. + let down_reason = if status != StatusCode::OK { + let code = status.as_u16(); + Some(match go_status_text(status) { + Some(text) => format!("the ping request returned HTTP {code} {text}"), + None => format!("the ping request returned HTTP {code}"), + }) + } else if !body.is_empty() { + Some("the ping request returned a non-empty body".to_string()) + } else { + None + }; ServerStatus { - up: status == StatusCode::OK, - tls: facts.tls, + up: down_reason.is_none(), + down_reason, + // As before, a probe answered with a body reports nothing + // about its connection. + tls: facts.tls.filter(|_| body.is_empty()), } } Err(e) => { write_debug!(out, "Error checking for server status: {e:#}\n"); - ServerStatus::default() + ServerStatus::down(format!("{e:#}")) } } } @@ -615,16 +630,43 @@ fn format_rate(label: &str, use_bytes: bool, counter: &BytesCounter) -> String { } } -/// How much of a probe response is worth keeping to quote in a diagnostic. -const PROBE_BODY_PEEK: usize = 8 * 1024; +/// How much of a probe response is read. Any body at all means the backend is +/// down, and the Go client reads and quotes the same first 512 bytes. +const PROBE_BODY_PEEK: usize = 512; + +/// The reason phrase Go's `http.StatusText` gives a status, as the Go client +/// prints it rather than the one the server sent. The http crate words four +/// codes differently, and for a code neither knows there is none. +fn go_status_text(status: StatusCode) -> Option<&'static str> { + match status.as_u16() { + 203 => Some("Non-Authoritative Information"), + 413 => Some("Request Entity Too Large"), + 414 => Some("Request URI Too Long"), + 416 => Some("Requested Range Not Satisfiable"), + _ => status.canonical_reason(), + } +} /// Whether a backend answered its probe, and what that connection negotiated. #[derive(Debug, Default)] pub struct ServerStatus { pub up: bool, + /// Why the backend is down, when it is: the refused connection, the + /// unreachable network, the wrong status or a body where none belongs. + pub down_reason: Option, pub tls: Option, } +impl ServerStatus { + fn down(reason: String) -> Self { + ServerStatus { + up: false, + down_reason: Some(reason), + tls: None, + } + } +} + /// A running `--json-stream` progress ticker. struct ProgressTicker { stop: Arc, @@ -940,6 +982,25 @@ mod tests { fn sponsor_is_empty_without_a_name() { assert_eq!(Server::default().sponsor(Output::default()), ""); } + + // Go's net/http status table, where it differs from the http crate's, a + // few where they agree, and a code neither knows. + #[test] + fn status_text_is_gos() { + for (code, text) in [ + (203, Some("Non-Authoritative Information")), + (404, Some("Not Found")), + (413, Some("Request Entity Too Large")), + (414, Some("Request URI Too Long")), + (416, Some("Requested Range Not Satisfiable")), + (422, Some("Unprocessable Entity")), + (500, Some("Internal Server Error")), + (599, None), + ] { + let status = StatusCode::from_u16(code).unwrap(); + assert_eq!(go_status_text(status), text, "{code}"); + } + } } #[cfg(test)] diff --git a/src/helper.rs b/src/helper.rs index c9c4a3a..ee9a75a 100644 --- a/src/helper.rs +++ b/src/helper.rs @@ -55,6 +55,12 @@ pub async fn do_speed_test( servers: &[Server], ctx: &TestContext<'_>, ) -> anyhow::Result<()> { + // With nothing to test the report would be empty, like when every server + // is down (--server -1 with an empty list gets here). + if servers.is_empty() { + anyhow::bail!("No server to test: the server list is empty"); + } + let out = ctx.out; if servers.len() > 1 { write_ui!(out, "Testing against {} servers\n", servers.len()); @@ -63,6 +69,9 @@ pub async fn do_speed_test( let delimiter = cli.csv_delimiter_byte()?; let mut reps_json: Vec = Vec::new(); let mut reps_csv: Vec = Vec::new(); + let mut measured = false; + // Each server that did not answer its probe, and why. + let mut down: Vec<(String, String)> = Vec::new(); for current_server in servers { let tlog = TelemetryLog::new(); @@ -106,6 +115,16 @@ pub async fn do_speed_test( if servers.len() > 1 && !out.quiet { output::write_ui_blank(); } + down.push(( + format!( + "{} ({})", + output::sanitize(¤t_server.name), + output::sanitize(&hostname) + ), + status + .down_reason + .unwrap_or_else(|| "its probe failed".to_string()), + )); continue; } @@ -339,12 +358,20 @@ pub async fn do_speed_test( }); } + measured = true; + // Add a blank line after each test when testing multiple servers. if servers.len() > 1 && !out.quiet { output::write_ui_blank(); } } + // No server answered: fail instead of printing an empty report, which with + // exit status 0 would pass for a result. + if !measured && !down.is_empty() { + return Err(none_responding(&down)); + } + if cli.csv { match report::csv_rows(&reps_csv, delimiter) { Ok(s) => write_out!("{s}"), @@ -377,11 +404,25 @@ fn report_failure(what: &str, e: anyhow::Error) -> anyhow::Error { e } +/// The error for a run whose servers were all down: each one and its cause. +fn none_responding(down: &[(String, String)]) -> anyhow::Error { + match down { + [(server, why)] => anyhow::anyhow!("Selected server {server} is not responding: {why}"), + _ => anyhow::anyhow!( + "None of the {} selected servers is responding: {}", + down.len(), + down.iter() + .map(|(server, why)| format!("{server}: {why}")) + .collect::>() + .join("; ") + ), + } +} + /// The reports as JSON sees them: `null` when nothing was measured. /// -/// Go declares a nil slice, which marshals to `null`, and both clients exit 0 -/// even when every server failed -- so the document is the only signal that -/// nothing happened, and a consumer testing for null must keep seeing it. +/// Go declares a nil slice, which marshals to `null`. No run gets here with +/// nothing measured any more: down servers and an empty list fail it first. fn json_reports(reports: &[JSONReport]) -> Option<&[JSONReport]> { (!reports.is_empty()).then_some(reports) } diff --git a/tests/data/parity/servers-empty.json b/tests/data/parity/servers-empty.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/tests/data/parity/servers-empty.json @@ -0,0 +1 @@ +[] diff --git a/tests/integration.rs b/tests/integration.rs index 65f4f5d..1bb53b8 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -149,6 +149,18 @@ fn handle(mut stream: TcpStream, telemetry_hits: Arc) -> std::io::R stream.write_all(body)?; stream.flush() }; + // What a real web server answers for a missing file or a broken script: + // the status, with an error page. + let error_page = |stream: &mut TcpStream, status: &str| -> std::io::Result<()> { + let page = format!("

{status}

\n"); + let head = format!( + "HTTP/1.1 {status}\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + page.len() + ); + stream.write_all(head.as_bytes())?; + stream.write_all(page.as_bytes())?; + stream.flush() + }; match (method.as_str(), path.as_str()) { (_, "/empty.php") => respond(&mut stream, b"", "text/plain")?, @@ -176,6 +188,12 @@ fn handle(mut stream: TcpStream, telemetry_hits: Arc) -> std::io::R telemetry_hits.fetch_add(1, Ordering::Relaxed); respond(&mut stream, b"id 4815162342", "text/plain")? } + ("GET", "/error-page.php") => error_page(&mut stream, "404 Not Found")?, + ("GET", "/server-error.php") => error_page(&mut stream, "500 Internal Server Error")?, + // A newer reason phrase than Go's, and a code with none: the error + // names neither the way the server put it. + ("GET", "/too-large.php") => error_page(&mut stream, "413 Content Too Large")?, + ("GET", "/unknown-status.php") => error_page(&mut stream, "599 Made Up")?, _ => { stream.write_all( b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", @@ -1003,3 +1021,269 @@ fn a_failed_step_is_reported_with_its_cause() { "{stderr}" ); } + +/// A URL whose port nothing listens on, below the ephemeral range as with the +/// parity fixture's dead port, so no other test's socket can be given it. +fn dead_url() -> String { + let first = 20000 + (std::process::id() % 5000) as u16; + let port = (first..30000) + .find(|port| TcpListener::bind(("127.0.0.1", *port)).is_ok()) + .expect("a free port"); + format!("http://127.0.0.1:{port}/") +} + +/// Writes a server list and returns its path; each entry is `(id, name, +/// server URL, ping URL)`. +fn write_server_list(name: &str, servers: &[(i64, &str, String, &str)]) -> PathBuf { + let path = std::env::temp_dir().join(format!( + "librespeed-cli-test-{name}-{}.json", + std::process::id() + )); + let entries: Vec = servers + .iter() + .map(|(id, name, url, ping)| { + format!( + r#"{{"name":"{name}","server":"{url}","id":{id},"dlURL":"garbage.php","ulURL":"empty.php","pingURL":"{ping}","getIpURL":"getIP.php","sponsorName":"","sponsorURL":""}}"# + ) + }) + .collect(); + std::fs::write(&path, format!("[{}]", entries.join(","))).expect("write server list"); + path +} + +/// The error the run terminated with, from the line main prints for it; panics +/// when stderr has no such line. +fn failure(out: &Output) -> String { + const PREFIX: &str = "Terminated due to error: "; + let stderr = String::from_utf8_lossy(&out.stderr); + stderr + .lines() + .find_map(|line| line.strip_prefix(PREFIX)) + .unwrap_or_else(|| panic!("stderr has no {PREFIX:?} line:\n{stderr}")) + .to_string() +} + +// An unreachable server fails the run in every mode: exit 1, the server and the +// cause on stderr, no report on stdout, as with the Go client. +#[test] +fn a_server_that_is_down_fails_the_run_in_every_mode() { + const DOWN: &str = "Selected server Dead (127.0.0.1) is not responding: "; + let list = write_server_list("down", &[(1, "Dead", dead_url(), "empty.php")]); + for mode in [ + &[][..], + &["--json"], + &["--json-stream"], + &["--csv"], + &["--simple"], + ] { + let mut args = vec!["--local-json", list.to_str().unwrap(), "--server", "1"]; + args.extend_from_slice(&["--duration", "1", "--no-icmp"]); + args.extend_from_slice(mode); + let out = run(&args); + + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!(out.status.code(), Some(1), "{mode:?}: {stderr}"); + assert_eq!(stdout_of(&out), "", "{mode:?}: no report"); + let msg = failure(&out); + // The cause is worded by reqwest and the OS; only its presence counts. + let cause = msg + .strip_prefix(DOWN) + .unwrap_or_else(|| panic!("{mode:?}: {msg}")); + assert!(!cause.is_empty(), "{mode:?}: the cause is named: {msg}"); + if mode.is_empty() { + assert!( + stderr.contains( + "\nSelected server Dead (127.0.0.1) is not responding at the moment, try again later\n" + ), + "{stderr}" + ); + } + } +} + +// A backend answering its probe wrongly is down, and the error says what it +// answered: a wrong status (even with an error page), otherwise the body. +#[test] +fn a_server_answering_its_probe_wrongly_fails_the_run_with_what_it_answered() { + let backend = MockBackend::start(); + for (ping, answered) in [ + ("missing.php", "HTTP 404 Not Found"), + ("error-page.php", "HTTP 404 Not Found"), + ("server-error.php", "HTTP 500 Internal Server Error"), + ("too-large.php", "HTTP 413 Request Entity Too Large"), + ("unknown-status.php", "HTTP 599"), + ("getIP.php", "a non-empty body"), + ] { + let list = write_server_list("probe-wrong", &[(1, "Wrong", backend.url(), ping)]); + let out = run(&[ + "--local-json", + list.to_str().unwrap(), + "--server", + "1", + "--no-icmp", + "--json", + ]); + assert_eq!(out.status.code(), Some(1), "{ping}"); + assert_eq!(stdout_of(&out), "", "{ping}"); + assert_eq!( + failure(&out), + format!( + "Selected server Wrong (127.0.0.1) is not responding: the ping request returned {answered}" + ), + "{ping}" + ); + } +} + +// With several servers all down, the error names each one with its own cause, +// not only the last one tried. +#[test] +fn a_run_whose_servers_are_all_down_names_each_with_its_cause() { + let backend = MockBackend::start(); + let list = write_server_list( + "all-down", + &[ + (1, "Dead", dead_url(), "empty.php"), + (2, "Wrong", backend.url(), "error-page.php"), + ], + ); + let out = run(&[ + "--local-json", + list.to_str().unwrap(), + "--server", + "1", + "--server", + "2", + "--duration", + "1", + "--no-icmp", + "--json", + ]); + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!(out.status.code(), Some(1), "{stderr}"); + assert_eq!(stdout_of(&out), "", "no report"); + let msg = failure(&out); + let causes = msg + .strip_prefix("None of the 2 selected servers is responding: ") + .unwrap_or_else(|| panic!("{msg}")); + // Split at the last "; ": the first cause's wording is not ours. + let (dead, wrong) = causes.rsplit_once("; ").unwrap_or_else(|| panic!("{msg}")); + let refused = dead + .strip_prefix("Dead (127.0.0.1): ") + .unwrap_or_else(|| panic!("{msg}")); + assert!(!refused.is_empty(), "the refusal is named: {msg}"); + assert_eq!( + wrong, + "Wrong (127.0.0.1): the ping request returned HTTP 404 Not Found" + ); +} + +// With more than one server, a run that measured any of them reports those and +// succeeds, as the Go client's does. +#[test] +fn a_down_server_among_others_leaves_the_run_to_the_ones_that_answer() { + let backend = MockBackend::start(); + let list = write_server_list( + "one-down", + &[ + (1, "Live", backend.url(), "empty.php"), + (2, "Dead", dead_url(), "empty.php"), + ], + ); + let out = run(&[ + "--local-json", + list.to_str().unwrap(), + "--server", + "1", + "--server", + "2", + "--duration", + "0", + "--no-download", + "--no-upload", + "--no-icmp", + "--json", + ]); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + let reports: serde_json::Value = serde_json::from_str(&stdout_of(&out)).expect("valid JSON"); + let names: Vec<&str> = reports + .as_array() + .expect("a report array") + .iter() + .map(|r| r["server"]["name"].as_str().unwrap()) + .collect(); + assert_eq!(names, ["Live"]); +} + +// A run with no server to test, as with --server -1 and an empty list, fails +// in every mode like one whose servers are all down, without an empty report. +#[test] +fn a_run_with_no_server_to_test_fails() { + let list = write_server_list("empty", &[]); + for mode in [ + &[][..], + &["--json"], + &["--json-stream"], + &["--csv"], + &["--simple"], + ] { + let mut args = vec!["--local-json", list.to_str().unwrap(), "--server", "-1"]; + args.extend_from_slice(mode); + let out = run(&args); + + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!(out.status.code(), Some(1), "{mode:?}: {stderr}"); + assert_eq!(stdout_of(&out), "", "{mode:?}: no report"); + assert_eq!( + failure(&out), + "No server to test: the server list is empty", + "{mode:?}" + ); + } +} + +// Without --server the fastest server is picked and a down one skipped, as +// before: with every server down the run fails before any test. +#[test] +fn automatic_selection_with_every_server_down_fails_as_before() { + let backend = MockBackend::start(); + let list = write_server_list( + "auto-down", + &[ + (1, "Dead", dead_url(), "empty.php"), + (2, "Wrong", backend.url(), "error-page.php"), + ], + ); + let out = run(&[ + "--local-json", + list.to_str().unwrap(), + "--no-icmp", + "--json", + ]); + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!(out.status.code(), Some(1), "{stderr}"); + assert_eq!(stdout_of(&out), "", "no report"); + assert!( + stderr.contains("No server is currently available, please try again later.\n"), + "{stderr}" + ); +} + +// A server list that cannot be fetched fails as it always did. +#[test] +fn an_unreachable_server_list_still_fails_the_run() { + let url = format!("{}servers.json", dead_url()); + let out = run(&["--server-json", &url, "--server", "1", "--json"]); + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!(out.status.code(), Some(1), "{stderr}"); + assert_eq!(stdout_of(&out), ""); + assert!( + stderr.contains("Error when fetching server list: "), + "{stderr}" + ); + assert!(!failure(&out).is_empty(), "{stderr}"); +} diff --git a/tests/parity/cases.rs b/tests/parity/cases.rs index f2ee135..8c166da 100644 --- a/tests/parity/cases.rs +++ b/tests/parity/cases.rs @@ -11,7 +11,7 @@ use crate::compare::View; /// The commit of librespeed/speedtest-cli the expectations were recorded /// against. CI builds exactly this one; keep `.github/workflows/ci.yml` in step. -pub const GO_COMMIT: &str = "b660d1e6c24f14fc93624538d9e73163e7784335"; +pub const GO_COMMIT: &str = "8aefd77bd4bde428cd86d3622102f0ac0443d96e"; /// Why a difference is there: the words of README.md that explain it, so a /// difference nobody documented cannot be declared. @@ -193,6 +193,7 @@ const SCHEMELESS: &str = "@FIX@/servers-schemeless.json"; const SCHEMELESS_PORT: &str = "@FIX@/servers-schemeless-port.json"; const HOSTILE: &str = "@FIX@/servers-hostile.json"; const GARBLED: &str = "@FIX@/servers-garbled.json"; +const EMPTY: &str = "@FIX@/servers-empty.json"; /// Reaches the ping and the report without moving any data. const NO_TRANSFER: [&str; 6] = [ @@ -259,14 +260,33 @@ fn reports() -> Vec { /// A backend that is not there, and a telemetry server that is not either. fn failures() -> Vec { + // Both clients fail the run without a report and name the cause, each in + // its own libraries' words. let dead = |name, args: &[&'static str]| { let mut all = vec!["--duration", "1", "--server", "2"]; all.extend_from_slice(args); - listed(name, SERVERS, &all) + listed(name, SERVERS, &all).stderr( + &[Why::ErrorCauses], + &[ + "Terminated due to error: Selected server Fixture Dead (127.0.0.1) is not responding: Get \"http://127.0.0.1:/empty.php\": dial tcp 127.0.0.1:: connect: connection refused", + ], + &[ + "Terminated due to error: Selected server Fixture Dead (127.0.0.1) is not responding: client error (Connect): Connection refused (os error )", + ], + ) }; vec![ + dead("dead_plain", &[]), dead("dead_json", &["--json"]), + dead("dead_json_stream", &["--json-stream"]), dead("dead_csv", &["--csv"]), + // Nothing to test at all: --server -1 takes every server of a list + // that has none, and without --server there is nothing to pick from. + listed("empty_list_plain", EMPTY, &["--server", "-1"]), + listed("empty_list_json", EMPTY, &["--server", "-1", "--json"]), + listed("empty_list_json_stream", EMPTY, &["--server", "-1", "--json-stream"]), + listed("empty_list_csv", EMPTY, &["--server", "-1", "--csv"]), + listed("empty_list_autoselect", EMPTY, &["--json"]), listed("server_not_in_list", SERVERS, &["--duration", "1", "--server", "99"]), listed("server_not_found_two", SERVERS, &["--server", "99", "--server", "98"]), // No --server: every server is probed, and one of them is dead. diff --git a/tests/parity/main.rs b/tests/parity/main.rs index 9a8a114..6e787b7 100644 --- a/tests/parity/main.rs +++ b/tests/parity/main.rs @@ -17,7 +17,7 @@ //! //! ```sh //! git clone https://github.com/librespeed/speedtest-cli go-client -//! git -C go-client checkout b660d1e6c24f14fc93624538d9e73163e7784335 +//! git -C go-client checkout 8aefd77bd4bde428cd86d3622102f0ac0443d96e //! (cd go-client && go build -o librespeed-go .) //! LIBRESPEED_GO_BIN=$PWD/go-client/librespeed-go cargo test --test parity //! ``` @@ -53,7 +53,7 @@ const RUN_TIMEOUT: Duration = Duration::from_secs(120); /// normalized away, so this only has to stay below what a CI runner can carry. const WORKERS: usize = 4; -const SERVER_LISTS: [(&str, &str); 7] = [ +const SERVER_LISTS: [(&str, &str); 8] = [ ("servers.json", include_str!("../data/parity/servers.json")), ( "servers-special.json", @@ -75,6 +75,10 @@ const SERVER_LISTS: [(&str, &str); 7] = [ "servers-garbled.json", include_str!("../data/parity/servers-garbled.json"), ), + ( + "servers-empty.json", + include_str!("../data/parity/servers-empty.json"), + ), ("bad.json", include_str!("../data/parity/bad.json")), ];