diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index 7c71f686..db8b8862 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -10,6 +10,9 @@ publish.workspace = true [lints] workspace = true +[features] +e2e-test-hooks = ["rocm-engine-lemonade/e2e-test-hooks"] + [dependencies] anyhow.workspace = true clap.workspace = true diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 9674403f..a507dfc0 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -4853,8 +4853,13 @@ fn serve(args: ServeArgs) -> Result<()> { // BEFORE preparing or launching any engine (no wasted engine download, and an // actionable message instead of a late engine crash). The engine enforces the // same rule as a backstop. Skipped for cpu_only; permissive when availability - // cannot be probed on this platform (probe returns `None`). + // cannot be probed on this platform (probe returns `None`). The E2E-only + // backend-failure scenario bypasses this host precondition so the black-box + // test reaches Lemonade's backend boundary without real GPU hardware. + let scripted_backend_failure = cfg!(feature = "e2e-test-hooks") + && std::env::var_os("ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE").is_some(); if !cpu_only + && !scripted_backend_failure && let Some(usable) = rocm_core::usable_amd_gpu_indices() && usable.is_empty() { diff --git a/engines/lemonade/Cargo.toml b/engines/lemonade/Cargo.toml index d2c28fb7..94e37252 100644 --- a/engines/lemonade/Cargo.toml +++ b/engines/lemonade/Cargo.toml @@ -10,6 +10,9 @@ publish.workspace = true [lints] workspace = true +[features] +e2e-test-hooks = [] + [[bin]] name = "rocm-engine-lemonade" path = "src/main.rs" diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index 315a06c0..94090b46 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -40,6 +40,8 @@ const DEFAULT_MODEL_REPO_DIR: &str = "models--unsloth--Qwen3-4B-Instruct-2507-GG const DEFAULT_MODEL_GGUF: &str = "Qwen3-4B-Instruct-2507-Q4_K_M.gguf"; const LLAMACPP_RECIPE: &str = "llamacpp"; const ROCM_BACKEND_NAME: &str = "rocm"; +#[cfg(feature = "e2e-test-hooks")] +const BACKEND_INSTALL_FAILURE_TEST_ENV: &str = "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE"; /// Preferred llama.cpp backends, best first. Lemonade reports per-GPU support; /// we pick the highest-priority backend it considers supported on this host. /// GPU backends only — `cpu` is intentionally excluded so the router path never @@ -426,6 +428,14 @@ fn detect_response() -> DetectResponse { fn install_response(request: InstallRequest) -> Result { let paths = AppPaths::discover()?; paths.ensure()?; + // Debug builds expose a deterministic failure seam for the black-box CLI + // scenario that pins retry count and terminal recovery guidance. Keep it at + // the backend phase: the real defect happens after the embeddable is ready, + // and exercising it must not download or alter a runtime on the test host. + #[cfg(feature = "e2e-test-hooks")] + if std::env::var_os(BACKEND_INSTALL_FAILURE_TEST_ENV).is_some() { + install_llamacpp_backend_with_retry(|| bail!("scripted Lemonade backend install failure"))?; + } eprintln!( "Preparing Lemonade embeddable {}...", rocm_deps::LEMONADE_VERSION @@ -1136,6 +1146,25 @@ fn install_best_llamacpp_backend(manifest: &mut LemonadeInstallManifest) -> Resu /// Ask Lemonade which llama.cpp backends it supports on this GPU, choose the best /// one (`LLAMACPP_BACKEND_PRIORITY`), install it if necessary, and return its name. +/// Retry the backend install itself once, rather than the whole Lemonade runtime +/// preparation. The embeddable download already has bounded transport retries; +/// repeating that outer operation would redo deterministic failures and may +/// re-extract a healthy runtime. A backend subprocess can instead fail after a +/// completed download when its connection to lemond is interrupted, and a second +/// call can reuse the backend cache immediately without a delay. +fn install_llamacpp_backend_with_retry(mut install: impl FnMut() -> Result<()>) -> Result<()> { + match install() { + Ok(()) => Ok(()), + Err(first_error) => { + eprintln!("Lemonade backend installation failed; retrying once: {first_error:#}"); + install().with_context(|| { + "Lemonade backend installation failed again; run `rocm engines install \ + lemonade --reinstall` and then retry `rocm serve`" + }) + } + } +} + fn ensure_best_llamacpp_backend( manifest: &LemonadeInstallManifest, host: &str, @@ -1157,7 +1186,9 @@ fn ensure_best_llamacpp_backend( eprintln!("Using installed Lemonade {LLAMACPP_RECIPE}:{backend} backend."); } else { eprintln!("Installing Lemonade {LLAMACPP_RECIPE}:{backend} backend..."); - run_lemonade_backend_install(manifest, host, port, &backend, process_env)?; + install_llamacpp_backend_with_retry(|| { + run_lemonade_backend_install(manifest, host, port, &backend, process_env) + })?; } Ok(backend) } @@ -4364,6 +4395,65 @@ mod tests { dir } + #[test] + fn backend_install_succeeds_without_retry() { + let mut attempts = 0; + + install_llamacpp_backend_with_retry(|| { + attempts += 1; + Ok(()) + }) + .unwrap(); + + assert_eq!(attempts, 1); + } + + #[test] + fn backend_install_recovers_on_the_second_attempt() { + let mut attempts = 0; + + install_llamacpp_backend_with_retry(|| { + attempts += 1; + if attempts == 1 { + bail!("first backend connection was interrupted"); + } + Ok(()) + }) + .unwrap(); + + assert_eq!(attempts, 2); + } + + #[test] + fn backend_install_stops_after_one_retry_with_reinstall_guidance() { + let mut attempts = 0; + + let error = install_llamacpp_backend_with_retry(|| { + attempts += 1; + if attempts == 1 { + bail!("first backend connection was interrupted"); + } + bail!("second backend connection was interrupted"); + }) + .unwrap_err(); + let rendered = format!("{error:#}"); + + assert_eq!(attempts, 2); + assert!( + rendered.contains("second backend connection was interrupted"), + "{rendered}" + ); + assert!( + !rendered.contains("first backend connection was interrupted"), + "the terminal error must be the retry's failure: {rendered}" + ); + assert!( + rendered.contains("rocm engines install lemonade --reinstall"), + "{rendered}" + ); + assert!(rendered.contains("retry `rocm serve`"), "{rendered}"); + } + fn test_manifest(runtime_dir: PathBuf) -> LemonadeInstallManifest { LemonadeInstallManifest { env_id: "test".to_owned(), diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index a9edaab9..3bf7aa47 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -154,3 +154,15 @@ Feature: Model serving When the user serves a model pinned to a GPU index that does not exist Then serving is refused before any engine starts And the user is told that GPU index is unavailable + + # The failure is injected at Lemonade's backend-install boundary in debug/test + # builds, after the CLI has selected Lemonade but before any runtime download or + # machine mutation. That makes the user-visible retry and final recovery command + # deterministic on the blocking no-GPU lane rather than relying on a real 3 GiB + # transfer to fail at just the right moment. + @id:serve-lemonade-preparation-recovery @requires-no-gpu + Scenario: 15 - Repeated Lemonade preparation failure gives the user a recovery path + Given Lemonade preparation cannot complete + When the user serves a model with Lemonade + Then serving stops after one automatic retry + And the user is told how to reinstall Lemonade and retry serving diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 3b61f673..3e752ff5 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -72,6 +72,10 @@ pub struct E2eWorld { /// a run whose failure is already the expected outcome — see the relaunch /// budget in `setup_gpu_model`. pub expect_xfail: bool, + /// Extra environment for this scenario's next `rocm` invocation. A Given step + /// records a behavioral precondition here; the When step remains a plain user + /// action and consumes the fixture without exposing its mechanism in Gherkin. + pub command_env: Vec<(&'static str, std::ffi::OsString)>, /// The interactive dash/chat TUI spawned under a pseudo-terminal for this /// scenario, if any (see `e2e::tui_driver`). Torn down in `Drop` before the /// mock server and isolated directory so the child process never outlives @@ -188,6 +192,7 @@ impl Default for E2eWorld { legacy_rocm_path: None, serve_timeout_override: None, expect_xfail: false, + command_env: Vec::new(), tui: None, chat_use_mock: false, lifecycle: None, @@ -549,6 +554,29 @@ pub fn run_rocm_with_env( ) } +/// Run `rocm` with the behavioral fixture established by a Given step, then +/// consume it so it cannot leak into a later action in the same scenario. +pub fn run_rocm_with_scenario_env(world: &mut E2eWorld, args: &[&str]) -> (String, String, i32) { + let binary = rocm_binary(); + let mut cmd = std::process::Command::new(&binary); + cmd.args(args); + world.isolate_cmd(&mut cmd); + for (key, value) in std::mem::take(&mut world.command_env) { + cmd.env(key, value); + } + let output = cmd + .output() + .unwrap_or_else(|e| panic!("failed to run {binary}: {e}")); + let rc = output.status.code().unwrap_or(-1); + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + record_command(world.current_scenario.as_deref(), args, rc, &stdout); + ( + stdout, + String::from_utf8_lossy(&output.stderr).to_string(), + rc, + ) +} + /// Append one `rocm` invocation to `results/commands.jsonl` so the consolidated /// report can build a command × platform coverage table tied to real results. /// Best-effort: a recording failure must never fail a scenario. diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index ca7b9da3..c19dd078 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -698,6 +698,31 @@ async fn user_serves_vllm_capable_default(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +#[given("Lemonade preparation cannot complete")] +async fn lemonade_preparation_cannot_complete(world: &mut E2eWorld) { + world.command_env.push(( + "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE", + "repeated".into(), + )); +} + +#[when("the user serves a model with Lemonade")] +async fn user_serves_with_failing_lemonade_preparation(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + world, + &[ + "serve", + "Qwen3-0.6B-GGUF", + "--engine", + "lemonade", + "--managed", + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + #[when("the user sends a chat completion request")] async fn user_sends_completion(world: &mut E2eWorld) { crate::send_chat(world).await; @@ -757,6 +782,34 @@ async fn when_cli_reports_ready(world: &mut E2eWorld) { // ── Then ─────────────────────────────────────────────────────────── +#[then("serving stops after one automatic retry")] +async fn assert_lemonade_preparation_retry_is_bounded(world: &mut E2eWorld) { + let output = serve_output(world); + assert_ne!( + world.cli_rc, + Some(0), + "serve unexpectedly succeeded:\n{output}" + ); + assert_eq!( + output.matches("retrying once").count(), + 1, + "expected exactly one retry announcement:\n{output}" + ); +} + +#[then("the user is told how to reinstall Lemonade and retry serving")] +async fn assert_lemonade_recovery_guidance(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("rocm engines install lemonade --reinstall"), + "expected a forced-reinstall recovery command:\n{output}" + ); + assert!( + output.contains("retry `rocm serve`"), + "expected guidance to retry serving:\n{output}" + ); +} + #[then("an inference request succeeds immediately")] async fn assert_inference_succeeds_now(world: &mut E2eWorld) { // No extra wait: the CLI already reported ready, so inference must work now. diff --git a/xtask/src/e2e.rs b/xtask/src/e2e.rs index 87d242a0..05cf29bb 100644 --- a/xtask/src/e2e.rs +++ b/xtask/src/e2e.rs @@ -85,7 +85,16 @@ pub fn run(args: &[String]) -> Result<()> { if binaries.build_release { let status = Command::new(&cargo) - .args(["build", "--release", "-p", "rocm", "-p", "rocmd"]) + .args([ + "build", + "--release", + "-p", + "rocm", + "-p", + "rocmd", + "--features", + "rocm/e2e-test-hooks", + ]) .current_dir(&root) .status() .context("failed to run `cargo build --release -p rocm -p rocmd`")?;