diff --git a/tests/e2e-cucumber/features/automations.feature b/tests/e2e-cucumber/features/automations.feature new file mode 100644 index 00000000..79fed50c --- /dev/null +++ b/tests/e2e-cucumber/features/automations.feature @@ -0,0 +1,27 @@ +Feature: Automation watchers + + # `rocm automations enable/disable --mode ` toggles a built-in + # automation watcher in the CLI config and prints a confirmation. This feature + # pins only the enable/disable/mode-confirmation slice verified during the + # walkthrough; the broader automation behaviour is covered elsewhere. Config-only + # (no GPU, no network), so it runs on the mock lane every PR. + + @id:automations-enable-confirms-mode + Scenario: 1 - Enabling a watcher confirms its mode + Given a fresh CLI configuration + When the user enables an automation watcher in observe mode + Then the CLI confirms the watcher is enabled in observe mode + When the user re-enables the same watcher in propose mode + Then the CLI confirms the watcher is enabled in propose mode + + @id:automations-disable-confirmed + Scenario: 2 - Disabling a watcher is confirmed + Given an enabled automation watcher + When the user disables the watcher + Then the CLI confirms the watcher is disabled + + @id:automations-enable-unknown-refused + Scenario: 3 - Enabling an unknown watcher is refused + Given a fresh CLI configuration + When the user tries to enable a watcher that does not exist + Then the CLI refuses and names it as unknown diff --git a/tests/e2e-cucumber/features/config.feature b/tests/e2e-cucumber/features/config.feature new file mode 100644 index 00000000..f77e5d2a --- /dev/null +++ b/tests/e2e-cucumber/features/config.feature @@ -0,0 +1,69 @@ +Feature: Configuration mutations + + # `rocm config <...>` edits the CLI's own config file (JSON under the isolated + # config dir) and prints a one-line confirmation. These behaviours were verified + # correct during the walkthrough but had no scenario protecting them. They touch + # only the config file — no GPU, no network — so they run on the mock lane every + # PR, catching a regression in the config surface before the field does. + + @id:config-default-engine-set-and-cleared + Scenario: 1 - Setting and clearing the default engine is confirmed + Given a fresh CLI configuration + When the user sets the default engine + Then the CLI confirms the default engine was set + When the user clears the default engine + Then the CLI confirms the default engine was cleared + + @id:config-default-runtime-set-and-cleared + Scenario: 2 - Setting and clearing the default runtime is confirmed + Given a fresh CLI configuration + When the user sets the default runtime + Then the CLI confirms the default runtime was set + When the user clears the default runtime + Then the CLI confirms the default runtime was cleared + + @id:config-telemetry-mode-confirmed + Scenario: 3 - Choosing a telemetry mode is confirmed with its policy + Given a fresh CLI configuration + When the user turns telemetry off + Then the CLI confirms the telemetry mode and states the policy + + @id:config-permissions-mode-confirmed + Scenario: 4 - Choosing a permissions mode is confirmed + Given a fresh CLI configuration + When the user selects a permissions mode + Then the CLI confirms the permissions mode + + @id:config-set-engine-requires-target + Scenario: 5 - Configuring an engine requires a target + Given a fresh CLI configuration + When the user configures an engine without saying what to change + Then the CLI refuses and explains a target is required + When the user configures an engine with a runtime to use + Then the CLI confirms the engine configuration was updated + + @id:config-provider-enable-disable + Scenario: 6 - Enabling and disabling a cloud provider is confirmed + Given a fresh CLI configuration + When the user enables a cloud provider + Then the CLI confirms the provider is enabled for prompt sending + When the user disables that provider + Then the CLI confirms the provider is disabled + + @id:config-local-provider-not-toggleable + Scenario: 7 - The always-on local provider cannot be toggled as a cloud provider + Given a fresh CLI configuration + When the user tries to enable the local provider + Then the CLI refuses and explains the local provider is always enabled + + # @requires-os:linux: the premise is a secure store that cannot save. The Linux + # Secret Service is reached over D-Bus, which the step forces unreachable so the + # save deterministically fails; the Windows/macOS credential stores are always + # present and cannot be disabled the same way, so the failure premise only holds + # on Linux. The no-echo property it verifies is the security-relevant contract. + @id:config-provider-key-no-secret-storage @requires-os:linux + Scenario: 8 - Saving a provider key without secure storage fails without leaking the key + Given a machine with no secure secret storage + When the user saves a provider API key + Then the CLI reports it could not save the key securely + And the key value never appears in the output diff --git a/tests/e2e-cucumber/features/logs.feature b/tests/e2e-cucumber/features/logs.feature new file mode 100644 index 00000000..d12168ad --- /dev/null +++ b/tests/e2e-cucumber/features/logs.feature @@ -0,0 +1,26 @@ +Feature: Log inspection + + # `rocm logs` reads the CLI's own recorded command logs from the isolated data + # dir and can filter them by a search term or a service. These behaviours were + # verified correct during the walkthrough but had no scenario. They read planted + # log files only — no GPU, no network — so they run on the mock lane every PR. + # The search count is asserted by the number of MATCHING lines (deterministic), + # not the recent-line total, which also counts other log sources. + + @id:logs-search-reports-match-count + Scenario: 1 - Searching logs reports how many recent lines match + Given recorded command logs containing several lines about a topic + When the user searches the logs for that topic + Then the CLI reports the matching recent lines + + @id:logs-search-absent-term-no-matches + Scenario: 2 - Searching logs for an absent term reports no matches + Given recorded command logs containing several lines about a topic + When the user searches the logs for a term that appears nowhere + Then the CLI reports no matching lines + + @id:logs-service-and-search-conflict + Scenario: 3 - Asking for a service and a search term at once is refused + Given recorded command logs containing several lines about a topic + When the user asks for one service's logs and a search term together + Then the CLI refuses and explains only one may be used diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index a9edaab9..854b9bad 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -154,3 +154,13 @@ Feature: Model serving When the user serves a model pinned to a GPU index that does not exist Then serving is refused before any engine starts And the user is told that GPU index is unavailable + + # A runtime and an environment are two ways to pick what a serve runs against, + # and choosing both at once is ambiguous, so the CLI rejects the combination + # during argument parsing — before any engine or GPU work. No device needed, so + # this runs on the mock lane every PR. + @id:serve-runtime-and-env-selectors-conflict + Scenario: 15 - Selecting both a runtime and an environment at once is refused + When the user serves a model selecting both a runtime and an environment + Then serving is refused before any engine starts + And the user is told the two selectors cannot be combined diff --git a/tests/e2e-cucumber/features/networking.feature b/tests/e2e-cucumber/features/networking.feature index 4a6ca72f..b0de7049 100644 --- a/tests/e2e-cucumber/features/networking.feature +++ b/tests/e2e-cucumber/features/networking.feature @@ -28,3 +28,14 @@ Feature: Native HTTP networking And the model is registered with the CLI When the user sends a one-shot chat prompt through the CLI Then the CLI prints the assistant's reply + + # The EAI-7409 public-bind contract. Binding the server to a non-loopback + # interface exposes it to the network, so the CLI refuses unless the user + # explicitly opts in with `--allow-public-bind`. This check is pre-flight — the + # first thing `serve` does, before any engine or model work — so it needs no GPU + # and runs on the mock lane every PR. + @id:networking-public-bind-requires-opt-in + Scenario: 3 - Binding to a public interface without opt-in is refused up front + When the user serves a model bound to a public interface without allowing public binding + Then serving is refused before any engine starts + And the user is told to allow public binding first diff --git a/tests/e2e-cucumber/features/runtime_lifecycle.feature b/tests/e2e-cucumber/features/runtime_lifecycle.feature new file mode 100644 index 00000000..93793c0e --- /dev/null +++ b/tests/e2e-cucumber/features/runtime_lifecycle.feature @@ -0,0 +1,39 @@ +Feature: Runtime lifecycle state machine + + # `rocm runtimes activate/rollback/uninstall/import` move a runtime through its + # registry state machine. Only install/adopt/list were covered before; the + # activate/rollback/uninstall/import transitions were verified during the + # walkthrough but unprotected. The scenarios plant read-only (externally-sourced) + # runtimes in the isolated registry, so no SDK download or GPU is needed — they + # run on the mock lane every PR. Related EAI-7404. + + @id:runtime-activate-records-previous + Scenario: 1 - Activating a runtime records where it changed from + Given two registered runtimes and none active + When the user activates the first runtime + Then that runtime becomes active having changed from nothing + When the user activates the second runtime + Then that runtime becomes active having changed from the first + + @id:runtime-rollback-returns-to-previous + Scenario: 2 - Rolling back returns to the previously active runtime + Given two registered runtimes with the second active after the first + When the user rolls back + Then the first runtime is active again + + @id:runtime-uninstall-keeps-external-folder + Scenario: 3 - Uninstalling an externally-sourced runtime keeps its folder + Given a registered read-only runtime + When the user uninstalls that runtime + Then its registry entry is removed + And its external folder is left in place + + @id:runtime-import-rejects-duplicate-unless-replacing + Scenario: 4 - Importing a runtime, then rejecting a duplicate unless replacing + Given a runtime manifest to import + When the user imports the runtime + Then the runtime is registered as read-only + When the user imports the same runtime again + Then the CLI refuses because it already exists + When the user imports it again allowing replacement + Then the import succeeds diff --git a/tests/e2e-cucumber/features/update.feature b/tests/e2e-cucumber/features/update.feature new file mode 100644 index 00000000..15cd57b2 --- /dev/null +++ b/tests/e2e-cucumber/features/update.feature @@ -0,0 +1,14 @@ +Feature: Update report + + # `rocm update` (with no arguments) prints an update report: which managed + # runtimes have updates, plus the status of each update feed (CLI, engines, + # model recipes, runtimes). The walkthrough verified it correctly distinguishes + # published feeds from not-configured ones, but nothing pinned it. Run with no + # managed runtimes so the report needs no network — mock lane, every PR. + + @id:update-report-distinguishes-feed-status + Scenario: 1 - The update report distinguishes configured from not-configured feeds + Given a machine with no managed runtimes + When the user checks for updates + Then the report shows there are no managed runtimes to update + And it reports each update feed's status, marking unpublished feeds as not configured diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index ea7ea3d1..0b9974a0 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -18,16 +18,21 @@ use tempfile::TempDir; mod e2e { pub mod artifact_steps; + pub mod automations_steps; pub mod bench_steps; pub mod chat_steps; + pub mod config_steps; pub mod dash_steps; pub mod diagnose_steps; pub mod engines_steps; pub mod examine_steps; pub mod lifecycle_steps; + pub mod logs_steps; + pub mod runtime_lifecycle_steps; pub mod runtime_steps; pub mod serving_steps; pub mod tui_driver; + pub mod update_steps; } // ── World ────────────────────────────────────────────────────────── @@ -517,6 +522,56 @@ pub fn run_rocm_ok(world: &E2eWorld, args: &[&str]) -> String { stdout } +/// Like [`run_rocm`], but writes `stdin` to the child's standard input and sets +/// extra environment variables on the child. +/// +/// Used by scenarios that drive a command reading from stdin — e.g. `config +/// set-provider-key`, which reads the secret from stdin non-interactively. The +/// scenario can then assert on both the exit code and that the piped secret is +/// never echoed back. `envs` lets a scenario also control the child's environment +/// (e.g. point the secret store at an unreachable D-Bus so the save deterministically +/// fails), applied on top of the scenario's isolated config/data/cache env. +pub fn run_rocm_with_stdin( + world: &E2eWorld, + args: &[&str], + stdin: &str, + envs: &[(&str, &str)], +) -> (String, String, i32) { + use std::io::Write as _; + use std::process::Stdio; + + let binary = rocm_binary(); + let mut cmd = std::process::Command::new(&binary); + cmd.args(args); + world.isolate_cmd(&mut cmd); + for (key, value) in envs { + cmd.env(key, value); + } + cmd.stdin(Stdio::piped()); + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + let mut child = cmd + .spawn() + .unwrap_or_else(|e| panic!("failed to run {binary}: {e}")); + child + .stdin + .take() + .expect("child stdin was not piped") + .write_all(stdin.as_bytes()) + .expect("failed to write to child stdin"); + let output = child + .wait_with_output() + .unwrap_or_else(|e| panic!("failed to wait on {binary}: {e}")); + let rc = output.status.code().unwrap_or(-1); + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + record_command(world.current_scenario.as_deref(), args, rc, &stdout); + ( + stdout, + String::from_utf8_lossy(&output.stderr).to_string(), + rc, + ) +} + /// Like [`run_rocm`], but with extra environment variables set on the child. /// /// Used by scenarios that must control the device environment the CLI and engine diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs new file mode 100644 index 00000000..b0c9d6cf --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -0,0 +1,159 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for `rocm automations enable/disable`. Black-box against the isolated +//! config dir. `automations enable` would otherwise spawn a detached background +//! daemon (`rocm daemon`) on first enable, which both adds a nondeterministic +//! `helper:` line and leaks a process past the scenario. To keep the mock lane +//! hermetic, every scenario first plants an automation runtime-state marking the +//! daemon already running under THIS test process's (live) pid, so the CLI's +//! double-spawn guard skips the spawn. Contracts verified against the running +//! Linux binary (EAI-8072). Scoped to the enable/disable/mode slice; the broader +//! automations feature is covered separately. + +use cucumber::{given, then, when}; + +use crate::E2eWorld; + +/// A built-in watcher id (`BUILTIN_WATCHERS` in rocm-core); stable and always +/// present, so scenarios can enable/disable it without depending on host state. +const WATCHER: &str = "therock-update"; + +/// Plant an automation runtime-state that marks the background daemon already +/// running under the test harness's own (live) pid. `automations enable` guards +/// against a second spawn when the recorded daemon pid is a live process, so this +/// suppresses the detached-daemon spawn — no `helper:` line, no leaked process. +/// +/// Shared with the `a fresh CLI configuration` Given (in `config_steps`), which +/// calls this so every automations scenario starting from a fresh config is also +/// spawn-suppressed. Exposed `pub(crate)` for that single caller. +pub(crate) fn suppress_daemon_spawn(world: &E2eWorld) { + let root = world.isolated_root.as_ref().expect("no isolated root"); + let dir = root.path().join("data").join("automations"); + std::fs::create_dir_all(&dir).expect("failed to create automations dir"); + let state = serde_json::json!({ + "running": true, + "automations_enabled": true, + "daemon_pid": std::process::id(), + "started_at_unix_ms": 1_700_000_000_000u64, + "last_tick_unix_ms": 1_700_000_000_000u64, + "active_watchers": [], + }); + std::fs::write( + dir.join("runtime-state.json"), + serde_json::to_vec_pretty(&state).expect("failed to serialize automation state"), + ) + .expect("failed to write automation runtime state"); +} + +#[given("an enabled automation watcher")] +async fn enabled_watcher(world: &mut E2eWorld) { + suppress_daemon_spawn(world); + crate::run_rocm_ok( + world, + &["automations", "enable", WATCHER, "--mode", "observe"], + ); +} + +// ── When ─────────────────────────────────────────────────────────── + +#[when("the user enables an automation watcher in observe mode")] +async fn enable_observe(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &["automations", "enable", WATCHER, "--mode", "observe"], + ); + record(world, stdout, stderr, rc); +} + +#[when("the user re-enables the same watcher in propose mode")] +async fn enable_propose(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &["automations", "enable", WATCHER, "--mode", "propose"], + ); + record(world, stdout, stderr, rc); +} + +#[when("the user disables the watcher")] +async fn disable_watcher(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["automations", "disable", WATCHER]); + record(world, stdout, stderr, rc); +} + +#[when("the user tries to enable a watcher that does not exist")] +async fn enable_unknown(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &[ + "automations", + "enable", + "e2e-no-such-watcher", + "--mode", + "observe", + ], + ); + record(world, stdout, stderr, rc); +} + +// ── Then ─────────────────────────────────────────────────────────── + +#[then(regex = r"^the CLI confirms the watcher is enabled in (observe|propose) mode$")] +async fn confirm_enabled(world: &mut E2eWorld, mode: String) { + let out = ok_output(world); + assert!( + out.contains("automation watcher enabled"), + "expected an enable confirmation, got:\n{out}" + ); + assert!( + out.contains(&format!("watcher: {WATCHER}")), + "expected the watcher id, got:\n{out}" + ); + assert!( + out.contains(&format!("mode: {mode}")), + "expected mode {mode}, got:\n{out}" + ); +} + +#[then("the CLI confirms the watcher is disabled")] +async fn confirm_disabled(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains("automation watcher disabled") && out.contains(&format!("watcher: {WATCHER}")), + "expected a disable confirmation for {WATCHER}, got:\n{out}" + ); +} + +#[then("the CLI refuses and names it as unknown")] +async fn refuse_unknown(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + assert!(rc != 0, "expected refusal, got rc=0:\n{}", combined(world)); + assert!( + combined(world).contains("unknown watcher: e2e-no-such-watcher"), + "expected an unknown-watcher error, got:\n{}", + combined(world) + ); +} + +// ── Helpers ──────────────────────────────────────────────────────── + +fn record(world: &mut E2eWorld, stdout: String, stderr: String, rc: i32) { + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +fn combined(world: &E2eWorld) -> String { + format!( + "{}\n{}", + world.cli_output.as_deref().unwrap_or(""), + world.cli_stderr.as_deref().unwrap_or("") + ) +} + +fn ok_output(world: &E2eWorld) -> String { + let rc = world.cli_rc.expect("no command rc recorded"); + assert_eq!(rc, 0, "expected success, got rc={rc}:\n{}", combined(world)); + world.cli_output.clone().unwrap_or_default() +} diff --git a/tests/e2e-cucumber/tests/e2e/config_steps.rs b/tests/e2e-cucumber/tests/e2e/config_steps.rs new file mode 100644 index 00000000..726b3de0 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/config_steps.rs @@ -0,0 +1,258 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for `rocm config <...>` mutations. Black-box: each runs the real binary +//! against the scenario's isolated config dir and asserts on the printed +//! confirmation / exit code. Contracts verified against the running Linux binary +//! (see EAI-8072). No GPU or network — mock lane. + +use cucumber::{given, then, when}; + +use crate::E2eWorld; + +/// A distinctive fake key we pipe into `set-provider-key`; the negative scenario +/// asserts it is NEVER echoed back in stdout or stderr. +const FAKE_PROVIDER_KEY: &str = "e2e-secret-key-must-not-be-echoed"; + +#[given("a fresh CLI configuration")] +async fn fresh_config(world: &mut E2eWorld) { + // The World already gives each scenario an isolated, empty config dir. Also + // plant the automation daemon-suppression marker: this Given backs the + // automations scenarios too, and `automations enable` would otherwise spawn a + // detached daemon and leak it (see automations_steps::suppress_daemon_spawn). + // Harmless for pure-config scenarios, which never touch automation state. + crate::e2e::automations_steps::suppress_daemon_spawn(world); +} + +#[given("a machine with no secure secret storage")] +async fn no_secret_storage(_world: &mut E2eWorld) { + // The premise is made deterministic in the When step: the CLI is spawned with + // DBUS_SESSION_BUS_ADDRESS pointed at an unreachable socket, so the Linux + // Secret Service backend cannot connect and the save fails regardless of + // whether the runner happens to have a session bus. The scenario is + // `@requires-os:linux` because the Windows/macOS credential stores are not + // reachable through D-Bus and cannot be disabled the same way. +} + +// ── When ─────────────────────────────────────────────────────────── + +#[when("the user sets the default engine")] +async fn set_default_engine(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["config", "set-default-engine", "vllm"]); + record(world, stdout, stderr, rc); +} + +#[when("the user clears the default engine")] +async fn clear_default_engine(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["config", "clear-default-engine"]); + record(world, stdout, stderr, rc); +} + +#[when("the user sets the default runtime")] +async fn set_default_runtime(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &["config", "set-default-runtime", "therock-release:gfx942"], + ); + record(world, stdout, stderr, rc); +} + +#[when("the user clears the default runtime")] +async fn clear_default_runtime(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["config", "clear-default-runtime"]); + record(world, stdout, stderr, rc); +} + +#[when("the user turns telemetry off")] +async fn set_telemetry_off(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["config", "set-telemetry", "off"]); + record(world, stdout, stderr, rc); +} + +#[when("the user selects a permissions mode")] +async fn set_permissions(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["config", "set-permissions", "ask"]); + record(world, stdout, stderr, rc); +} + +#[when("the user configures an engine without saying what to change")] +async fn set_engine_no_target(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["config", "set-engine", "vllm"]); + record(world, stdout, stderr, rc); +} + +#[when("the user configures an engine with a runtime to use")] +async fn set_engine_with_runtime(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &[ + "config", + "set-engine", + "vllm", + "--runtime-id", + "therock-release:gfx942", + ], + ); + record(world, stdout, stderr, rc); +} + +#[when("the user enables a cloud provider")] +async fn enable_provider(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["config", "enable-provider", "openai"]); + record(world, stdout, stderr, rc); +} + +#[when("the user disables that provider")] +async fn disable_provider(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["config", "disable-provider", "openai"]); + record(world, stdout, stderr, rc); +} + +#[when("the user tries to enable the local provider")] +async fn enable_local_provider(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["config", "enable-provider", "local"]); + record(world, stdout, stderr, rc); +} + +#[when("the user saves a provider API key")] +async fn save_provider_key(world: &mut E2eWorld) { + // `set-provider-key` reads the key from stdin (non-interactive). Pipe a + // distinctive fake key so the "no echo" assertion can look for it. Force the + // Linux Secret Service unreachable by pointing D-Bus at a nonexistent socket, + // so the save deterministically fails on any Linux runner (see the Given). + let (stdout, stderr, rc) = crate::run_rocm_with_stdin( + world, + &["config", "set-provider-key", "openai"], + FAKE_PROVIDER_KEY, + &[( + "DBUS_SESSION_BUS_ADDRESS", + "unix:path=/nonexistent/e2e-no-bus", + )], + ); + record(world, stdout, stderr, rc); +} + +// ── Then ─────────────────────────────────────────────────────────── + +#[then("the CLI confirms the default engine was set")] +async fn confirm_engine_set(world: &mut E2eWorld) { + assert_ok_contains(world, "default engine set to vllm"); +} + +#[then("the CLI confirms the default engine was cleared")] +async fn confirm_engine_cleared(world: &mut E2eWorld) { + assert_ok_contains(world, "default engine cleared"); +} + +#[then("the CLI confirms the default runtime was set")] +async fn confirm_runtime_set(world: &mut E2eWorld) { + assert_ok_contains(world, "default runtime set to therock-release:gfx942"); +} + +#[then("the CLI confirms the default runtime was cleared")] +async fn confirm_runtime_cleared(world: &mut E2eWorld) { + assert_ok_contains(world, "default runtime cleared"); +} + +#[then("the CLI confirms the telemetry mode and states the policy")] +async fn confirm_telemetry(world: &mut E2eWorld) { + assert_ok_contains(world, "telemetry mode set to off"); + assert_output_contains(world, "policy:"); +} + +#[then("the CLI confirms the permissions mode")] +async fn confirm_permissions(world: &mut E2eWorld) { + assert_ok_contains(world, "permissions mode set to ask"); +} + +#[then("the CLI refuses and explains a target is required")] +async fn refuse_engine_no_target(world: &mut E2eWorld) { + assert_failed(world); + assert_output_contains( + world, + "set-engine requires --runtime-id, --env-id, or --clear", + ); +} + +#[then("the CLI confirms the engine configuration was updated")] +async fn confirm_engine_config_updated(world: &mut E2eWorld) { + assert_ok_contains(world, "updated engine config for vllm"); +} + +#[then("the CLI confirms the provider is enabled for prompt sending")] +async fn confirm_provider_enabled(world: &mut E2eWorld) { + assert_ok_contains(world, "provider openai enabled for prompt sending"); +} + +#[then("the CLI confirms the provider is disabled")] +async fn confirm_provider_disabled(world: &mut E2eWorld) { + assert_ok_contains(world, "provider openai disabled for prompt sending"); +} + +#[then("the CLI refuses and explains the local provider is always enabled")] +async fn refuse_local_provider(world: &mut E2eWorld) { + assert_failed(world); + assert_output_contains(world, "local provider is always enabled"); +} + +#[then("the CLI reports it could not save the key securely")] +async fn confirm_key_save_failed(world: &mut E2eWorld) { + assert_failed(world); + assert_output_contains(world, "failed to save openai API key in secure storage"); +} + +#[then("the key value never appears in the output")] +async fn confirm_key_not_echoed(world: &mut E2eWorld) { + let combined = combined_output(world); + assert!( + !combined.contains(FAKE_PROVIDER_KEY), + "the provider key leaked into the CLI output:\n{combined}" + ); +} + +// ── Helpers ──────────────────────────────────────────────────────── + +fn record(world: &mut E2eWorld, stdout: String, stderr: String, rc: i32) { + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +fn combined_output(world: &E2eWorld) -> String { + format!( + "{}\n{}", + world.cli_output.as_deref().unwrap_or(""), + world.cli_stderr.as_deref().unwrap_or("") + ) +} + +fn assert_output_contains(world: &E2eWorld, needle: &str) { + let combined = combined_output(world); + assert!( + combined.contains(needle), + "expected output to contain {needle:?}, got:\n{combined}" + ); +} + +/// Assert the last command succeeded (rc 0) and its output contains `needle`. +fn assert_ok_contains(world: &E2eWorld, needle: &str) { + let rc = world.cli_rc.expect("no command rc recorded"); + assert_eq!( + rc, + 0, + "expected success, got rc={rc}:\n{}", + combined_output(world) + ); + assert_output_contains(world, needle); +} + +/// Assert the last command failed (non-zero rc). +fn assert_failed(world: &E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + assert!( + rc != 0, + "expected the command to fail, but it exited 0:\n{}", + combined_output(world) + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/logs_steps.rs b/tests/e2e-cucumber/tests/e2e/logs_steps.rs new file mode 100644 index 00000000..ab0986d0 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/logs_steps.rs @@ -0,0 +1,125 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for `rocm logs`. Plant deterministic command-log files in the isolated +//! data dir, then assert on the search output. `logs` reads only the TAIL of each +//! file, so the matching lines are planted WITHIN the tail window and the +//! assertion is on the MATCH COUNT (deterministic), not the recent-line total +//! (which also counts other log sources). Contracts verified against the running +//! Linux binary (EAI-8072). No GPU or network — mock lane. + +use std::fmt::Write as _; + +use cucumber::{given, then, when}; + +use crate::E2eWorld; + +/// The topic term planted into the log and searched for. Distinctive so it can't +/// collide with anything the CLI itself writes into the isolated log dir. +const TOPIC: &str = "E2ENEEDLE"; +/// Number of matching lines planted — asserted exactly in the search result. +const MATCH_COUNT: usize = 9; + +#[given("recorded command logs containing several lines about a topic")] +async fn plant_command_logs(world: &mut E2eWorld) { + let root = world.isolated_root.as_ref().expect("no isolated root"); + let cli_logs = root.path().join("data").join("logs").join("cli"); + std::fs::create_dir_all(&cli_logs).expect("failed to create cli logs dir"); + // Write the matching lines LAST so they fall within the per-file tail window + // `rocm logs` reads (a few leading non-matching lines are harmless context). + let mut body = String::new(); + for i in 0..3 { + let _ = writeln!(body, "2026-01-01T00:00:0{i} unrelated startup line"); + } + for i in 1..=MATCH_COUNT { + let _ = writeln!( + body, + "2026-01-01T00:01:00 event {TOPIC} occurred number {i}" + ); + } + std::fs::write(cli_logs.join("e2e-probe.log"), body).expect("failed to write log file"); +} + +// ── When ─────────────────────────────────────────────────────────── + +#[when("the user searches the logs for that topic")] +async fn search_topic(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["logs", "--search", TOPIC]); + record(world, stdout, stderr, rc); +} + +#[when("the user searches the logs for a term that appears nowhere")] +async fn search_absent(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["logs", "--search", "e2e-no-such-term"]); + record(world, stdout, stderr, rc); +} + +#[when("the user asks for one service's logs and a search term together")] +async fn service_and_search(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &["logs", "--service", "some-service", "--search", TOPIC], + ); + record(world, stdout, stderr, rc); +} + +// ── Then ─────────────────────────────────────────────────────────── + +#[then("the CLI reports the matching recent lines")] +async fn reports_matches(world: &mut E2eWorld) { + let out = ok_output(world); + // The count of matching lines is deterministic (we planted exactly nine within + // the tail); the "of " denominator is not, so assert only the match side. + assert!( + out.contains(&format!("Lines: {MATCH_COUNT} of ")), + "expected {MATCH_COUNT} matching lines, got:\n{out}" + ); + assert!( + out.contains(&format!("Showing: 1-{MATCH_COUNT} of {MATCH_COUNT}")), + "expected the {MATCH_COUNT} matches to be listed, got:\n{out}" + ); +} + +#[then("the CLI reports no matching lines")] +async fn reports_no_matches(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains("Lines: 0 of ") && out.contains("Showing: 0 of 0"), + "expected no matching lines, got:\n{out}" + ); +} + +#[then("the CLI refuses and explains only one may be used")] +async fn refuses_conflict(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + assert!(rc != 0, "expected refusal, got rc=0:\n{}", combined(world)); + assert!( + combined(world) + .contains("accepts either --service or a search query, not both"), + "expected the service/search conflict message, got:\n{}", + combined(world) + ); +} + +// ── Helpers ──────────────────────────────────────────────────────── + +fn record(world: &mut E2eWorld, stdout: String, stderr: String, rc: i32) { + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +fn combined(world: &E2eWorld) -> String { + format!( + "{}\n{}", + world.cli_output.as_deref().unwrap_or(""), + world.cli_stderr.as_deref().unwrap_or("") + ) +} + +fn ok_output(world: &E2eWorld) -> String { + let rc = world.cli_rc.expect("no command rc recorded"); + assert_eq!(rc, 0, "expected success, got rc={rc}:\n{}", combined(world)); + world.cli_output.clone().unwrap_or_default() +} diff --git a/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs new file mode 100644 index 00000000..b525face --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs @@ -0,0 +1,281 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for the runtime state machine: `runtimes activate/rollback/uninstall/ +//! import`. Black-box against the isolated runtimes registry. A real SDK runtime +//! needs a multi-GiB download and a GPU family, so instead these plant READ-ONLY +//! `tarball` runtime manifests: the CLI validates a read-only tarball runtime by +//! only requiring its `install_root` to be a directory holding a non-dot payload +//! file — no python, no GPU, no download. That makes the whole state machine +//! exercisable on the mock lane. Contracts verified against the running Linux +//! binary (EAI-8072). Related EAI-7404. + +use std::path::{Path, PathBuf}; + +use cucumber::{given, then, when}; + +use crate::E2eWorld; + +const FIRST_KEY: &str = "therock-release:gfx942"; +const SECOND_KEY: &str = "therock-release:gfx1100"; +const IMPORT_KEY: &str = "therock-release:gfx1151"; + +/// Write a read-only `tarball` runtime manifest into the isolated registry and +/// create its `install_root` (a dir with a payload file) so it validates as usable. +/// Returns the install_root so a scenario can assert the folder's fate. +fn plant_runtime(world: &E2eWorld, key: &str, family: &str) -> PathBuf { + let root = world.isolated_root.as_ref().expect("no isolated root"); + let install_root = root.path().join(format!("runtime-{family}")); + std::fs::create_dir_all(&install_root).expect("failed to create install root"); + std::fs::write(install_root.join("payload.txt"), "payload") + .expect("failed to write runtime payload"); + + let registry = root.path().join("data").join("runtimes").join("registry"); + std::fs::create_dir_all(®istry).expect("failed to create registry dir"); + let manifest = runtime_manifest_json(key, family, &install_root); + std::fs::write(registry.join(format!("{key}.json")), manifest) + .expect("failed to write runtime manifest"); + install_root +} + +/// A minimal valid read-only tarball runtime manifest (matches the CLI's on-disk +/// schema). Written as plain JSON — black-box, not a typed import from the crates. +fn runtime_manifest_json(key: &str, family: &str, install_root: &Path) -> String { + serde_json::to_string_pretty(&serde_json::json!({ + "runtime_key": key, + "runtime_id": key, + "channel": "release", + "format": "tarball", + "family": family, + "family_source": "manual", + "version": "1.0.0", + "install_root": install_root, + "selected_artifact_url": format!("https://example.invalid/{key}.tar.gz"), + "read_only": true, + "installed_at_unix_ms": 1_700_000_000_000u64, + })) + .expect("failed to serialize runtime manifest") +} + +/// Path to an importable manifest file (not yet in the registry) for the import +/// scenario, with its install_root created so the import validates. +fn write_import_manifest(world: &E2eWorld) -> PathBuf { + let root = world.isolated_root.as_ref().expect("no isolated root"); + let install_root = root.path().join("runtime-import"); + std::fs::create_dir_all(&install_root).expect("failed to create import install root"); + std::fs::write(install_root.join("payload.txt"), "payload") + .expect("failed to write import payload"); + let manifest_path = root.path().join("import-manifest.json"); + std::fs::write( + &manifest_path, + runtime_manifest_json(IMPORT_KEY, "gfx1151", &install_root), + ) + .expect("failed to write import manifest"); + manifest_path +} + +// ── Given ────────────────────────────────────────────────────────── + +#[given("two registered runtimes and none active")] +async fn two_runtimes(world: &mut E2eWorld) { + plant_runtime(world, FIRST_KEY, "gfx942"); + plant_runtime(world, SECOND_KEY, "gfx1100"); +} + +#[given("two registered runtimes with the second active after the first")] +async fn two_runtimes_second_active(world: &mut E2eWorld) { + plant_runtime(world, FIRST_KEY, "gfx942"); + plant_runtime(world, SECOND_KEY, "gfx1100"); + // Activate first, then second, so `previous_runtime_key` records the first — + // the state rollback must return to. + crate::run_rocm_ok(world, &["runtimes", "activate", FIRST_KEY]); + crate::run_rocm_ok(world, &["runtimes", "activate", SECOND_KEY]); +} + +#[given("a registered read-only runtime")] +async fn one_readonly_runtime(world: &mut E2eWorld) { + let install_root = plant_runtime(world, FIRST_KEY, "gfx942"); + // Stash the install_root path so the uninstall scenario can assert it survives. + world.model_name = Some(install_root.to_string_lossy().into_owned()); +} + +#[given("a runtime manifest to import")] +async fn manifest_to_import(world: &mut E2eWorld) { + let path = write_import_manifest(world); + world.model_name = Some(path.to_string_lossy().into_owned()); +} + +// ── When ─────────────────────────────────────────────────────────── + +#[when("the user activates the first runtime")] +async fn activate_first(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["runtimes", "activate", FIRST_KEY]); + record(world, stdout, stderr, rc); +} + +#[when("the user activates the second runtime")] +async fn activate_second(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["runtimes", "activate", SECOND_KEY]); + record(world, stdout, stderr, rc); +} + +#[when("the user rolls back")] +async fn rollback(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["runtimes", "rollback"]); + record(world, stdout, stderr, rc); +} + +#[when("the user uninstalls that runtime")] +async fn uninstall(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["runtimes", "uninstall", FIRST_KEY]); + record(world, stdout, stderr, rc); +} + +#[when("the user imports the runtime")] +async fn import(world: &mut E2eWorld) { + let path = world.model_name.clone().expect("no import manifest path"); + let (stdout, stderr, rc) = crate::run_rocm(world, &["runtimes", "import", &path]); + record(world, stdout, stderr, rc); +} + +#[when("the user imports the same runtime again")] +async fn import_again(world: &mut E2eWorld) { + let path = world.model_name.clone().expect("no import manifest path"); + let (stdout, stderr, rc) = crate::run_rocm(world, &["runtimes", "import", &path]); + record(world, stdout, stderr, rc); +} + +#[when("the user imports it again allowing replacement")] +async fn import_replace(world: &mut E2eWorld) { + let path = world.model_name.clone().expect("no import manifest path"); + let (stdout, stderr, rc) = crate::run_rocm(world, &["runtimes", "import", &path, "--replace"]); + record(world, stdout, stderr, rc); +} + +// ── Then ─────────────────────────────────────────────────────────── + +#[then("that runtime becomes active having changed from nothing")] +async fn active_changed_from_nothing(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains("runtime activated") && out.contains(&format!("runtime_key: {FIRST_KEY}")), + "expected {FIRST_KEY} activated, got:\n{out}" + ); + assert!( + out.contains("changed_from_runtime_key: "), + "expected no previous runtime, got:\n{out}" + ); +} + +#[then("that runtime becomes active having changed from the first")] +async fn active_changed_from_first(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains("runtime activated") && out.contains(&format!("runtime_key: {SECOND_KEY}")), + "expected {SECOND_KEY} activated, got:\n{out}" + ); + assert!( + out.contains(&format!("changed_from_runtime_key: {FIRST_KEY}")), + "expected previous runtime {FIRST_KEY}, got:\n{out}" + ); +} + +#[then("the first runtime is active again")] +async fn first_active_again(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains("runtime rolled back") && out.contains(&format!("runtime_key: {FIRST_KEY}")), + "expected rollback to {FIRST_KEY}, got:\n{out}" + ); +} + +#[then("its registry entry is removed")] +async fn registry_removed(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains("runtime removed") && out.contains("registry_removed:"), + "expected the registry entry removed, got:\n{out}" + ); + let root = world.isolated_root.as_ref().expect("no isolated root"); + let entry = root + .path() + .join("data") + .join("runtimes") + .join("registry") + .join(format!("{FIRST_KEY}.json")); + assert!( + !entry.exists(), + "registry entry still present: {}", + entry.display() + ); +} + +#[then("its external folder is left in place")] +async fn folder_left(world: &mut E2eWorld) { + let out = world.cli_output.clone().unwrap_or_default(); + assert!( + out.contains("folder_removed: no") + && out.contains("existing external runtime folder was left untouched"), + "expected the external folder to be left, got:\n{out}" + ); + let install_root = world + .model_name + .as_deref() + .expect("no install root recorded"); + assert!( + Path::new(install_root).is_dir(), + "external runtime folder was removed: {install_root}" + ); +} + +#[then("the runtime is registered as read-only")] +async fn imported_readonly(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains("runtime imported") && out.contains("mode: read-only"), + "expected a read-only import, got:\n{out}" + ); +} + +#[then("the CLI refuses because it already exists")] +async fn import_duplicate_refused(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + assert!(rc != 0, "expected refusal, got rc=0:\n{}", combined(world)); + assert!( + combined(world).contains("already exists") && combined(world).contains("--replace"), + "expected a duplicate-registry error mentioning --replace, got:\n{}", + combined(world) + ); +} + +#[then("the import succeeds")] +async fn import_succeeds(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains("runtime imported"), + "expected the replace import to succeed, got:\n{out}" + ); +} + +// ── Helpers ──────────────────────────────────────────────────────── + +fn record(world: &mut E2eWorld, stdout: String, stderr: String, rc: i32) { + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +fn combined(world: &E2eWorld) -> String { + format!( + "{}\n{}", + world.cli_output.as_deref().unwrap_or(""), + world.cli_stderr.as_deref().unwrap_or("") + ) +} + +fn ok_output(world: &E2eWorld) -> String { + let rc = world.cli_rc.expect("no command rc recorded"); + assert_eq!(rc, 0, "expected success, got rc={rc}:\n{}", combined(world)); + world.cli_output.clone().unwrap_or_default() +} diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index ca7b9da3..46a50f31 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -742,6 +742,60 @@ async fn user_serves_absent_gpu_index(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +/// Serve bound to a public (non-loopback) interface without the public-bind +/// opt-in. The bind-host validation is the first thing `serve` does — before any +/// engine, model, or GPU work — so this refusal is observable with no GPU (mock +/// lane). The model name is arbitrary; the refusal happens before it is resolved. +#[when("the user serves a model bound to a public interface without allowing public binding")] +async fn user_serves_public_bind_no_optin(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm(world, &["serve", "some-model", "--host", "0.0.0.0"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +/// Serve naming both a runtime and an environment. These are mutually exclusive +/// selectors, rejected during argument parsing before any engine or GPU work — so +/// the refusal is observable with no GPU (mock lane). +#[when("the user serves a model selecting both a runtime and an environment")] +async fn user_serves_runtime_and_env(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &[ + "serve", + "some-model", + "--runtime-id", + "therock-release:gfx942", + "--env-id", + "some-env", + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the user is told to allow public binding first")] +async fn assert_public_bind_message(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("--allow-public-bind"), + "expected guidance to pass --allow-public-bind, got:\n{output}" + ); +} + +#[then("the user is told the two selectors cannot be combined")] +async fn assert_selector_conflict_message(world: &mut E2eWorld) { + let output = serve_output(world); + // clap's conflict error names both flags; assert both appear so a reworded + // message that still identifies the conflict keeps passing. + assert!( + output.contains("--runtime-id") && output.contains("--env-id"), + "expected a conflict naming --runtime-id and --env-id, got:\n{output}" + ); +} + #[when("the CLI reports the service as ready")] async fn when_cli_reports_ready(world: &mut E2eWorld) { // Read readiness from the CLI's own view (`services list`), not a direct diff --git a/tests/e2e-cucumber/tests/e2e/update_steps.rs b/tests/e2e-cucumber/tests/e2e/update_steps.rs new file mode 100644 index 00000000..acb0cdc7 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/update_steps.rs @@ -0,0 +1,78 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for `rocm update` (report only). Run with NO managed runtimes so the +//! report needs no network (with a runtime present, `update` reaches the TheRock +//! index to resolve the latest version). The report's update-feed status block is +//! host-invariant and is what pins the "distinguishes configured from +//! not-configured feeds" behaviour. Contracts verified against the running Linux +//! binary (EAI-8072). Mock lane. + +use cucumber::{given, then, when}; + +use crate::E2eWorld; + +#[given("a machine with no managed runtimes")] +async fn no_managed_runtimes(_world: &mut E2eWorld) { + // The World's isolated data dir starts with an empty runtimes registry, so + // `update` has nothing to check against the network. No setup required. +} + +#[when("the user checks for updates")] +async fn check_updates(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["update"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the report shows there are no managed runtimes to update")] +async fn no_runtimes_to_update(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains("managed runtimes: none"), + "expected 'managed runtimes: none', got:\n{out}" + ); +} + +#[then("it reports each update feed's status, marking unpublished feeds as not configured")] +async fn reports_feed_status(world: &mut E2eWorld) { + let out = ok_output(world); + // The update_surfaces block reports one line per feed. Assert each feed's status + // ON ITS OWN LINE, so a status attributed to the wrong feed fails — a check that + // only looked for the substrings anywhere would pass even if `not_configured` + // and `package_managed` were swapped between the cli and engines feeds. The CLI + // feed is not published yet (the "not configured" side of the distinction); + // engines and recipes report their own stable states. + for (feed, status) in [ + ("cli:", "status=not_configured"), + ("engines:", "status=package_managed"), + ("model_recipes:", "status=built_in"), + ] { + let line = out + .lines() + .map(str::trim) + .find(|line| line.starts_with(feed)); + match line { + Some(line) => assert!( + line.contains(status), + "update feed {feed:?} did not report {status:?} on its own line; got {line:?}\n\nfull output:\n{out}" + ), + None => panic!("no update feed line for {feed:?} in:\n{out}"), + } + } +} + +// ── Helpers ──────────────────────────────────────────────────────── + +fn ok_output(world: &E2eWorld) -> String { + let rc = world.cli_rc.expect("no command rc recorded"); + let combined = format!( + "{}\n{}", + world.cli_output.as_deref().unwrap_or(""), + world.cli_stderr.as_deref().unwrap_or("") + ); + assert_eq!(rc, 0, "expected success, got rc={rc}:\n{combined}"); + world.cli_output.clone().unwrap_or_default() +}