diff --git a/.github/workflows/e2e-windows.yml b/.github/workflows/e2e-windows.yml index 994d163..9b63b64 100644 --- a/.github/workflows/e2e-windows.yml +++ b/.github/workflows/e2e-windows.yml @@ -37,6 +37,8 @@ jobs: try { & ./tests/e2e-windows/run.ps1 } finally { Stop-Transcript } - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: always() + # The native journey above is the gate; diagnostics must not override its result. + continue-on-error: true with: name: windows-cli-e2e path: tests/e2e-windows/artifacts diff --git a/tests/e2e-slim/README.md b/tests/e2e-slim/README.md index ba651e3..19408b7 100644 --- a/tests/e2e-slim/README.md +++ b/tests/e2e-slim/README.md @@ -32,7 +32,7 @@ mutable or retired Docker Hub tags. | `all_agents_config` | The **same** model + MCP + skill assertions for `codex` / `kimi` / `opencode`, so all four agents are covered identically. | yes (per agent) | no | | `session_upload` | `blue session-upload` per harness → detached worker presigns → PUTs to MinIO → completes; the list shows all 4, and one artifact is downloaded and byte-compared to the upload. | no | no | | `agent_matrix_config` | **Tier A (version matrix).** The same model + MCP + skill assertions as `all_agents_config`, but per `(agent, version)` cell across multiple CLI versions (not just the pin). No inference. Self-skips unless the matrix is enabled. | yes (per cell) | no | -| `agent_certification` | **Tier B (version matrix).** Per `(agent, version)` cell, one uniform body: `blue run ` launches the real agent through the real gateway, forces the managed `blue_certify` MCP tool (asserts `mcp-started` + `mcp-called` markers and `BLUE_MCP_OK`), and asserts a LiteLLM spend log recorded the governed model alias with a swapped virtual key rather than the client inference JWT. With the matrix off it certifies just the pin per agent (today's coverage). | yes (per cell) | **yes** (opt-in) | +| `agent_certification` | **Tier B (version matrix).** Per `(agent, version)` cell, one uniform body: `blue run ` launches the real agent through the real gateway and forces the managed `blue_certify` MCP tool. Codex certification inspects its structured JSONL event stream for a completed `e2e-remote/blue_certify` call containing `BLUE_MCP_OK`; Claude, Kimi, and OpenCode assert the `mcp-started` + `mcp-called` marker files and `BLUE_MCP_OK`. The test also asserts that a LiteLLM spend log recorded the governed model alias with a swapped virtual key rather than the client inference JWT. With the matrix off it certifies just the pin per agent (today's coverage). | yes (per cell) | **yes** (opt-in) | **Real vs. faked.** Real: control-api, Postgres, MinIO (real presigned PUT/GET), the `blue` CLI, the agent CLIs (really installed, version-probed for eligibility), diff --git a/tests/e2e-slim/tests/agent_certification.rs b/tests/e2e-slim/tests/agent_certification.rs index 872eb82..40f15ee 100644 --- a/tests/e2e-slim/tests/agent_certification.rs +++ b/tests/e2e-slim/tests/agent_certification.rs @@ -9,11 +9,12 @@ //! Each `_certifies` test: //! 1. `blue apply --yes` — asserts the managed `e2e-remote` MCP server and the //! managed `example` skill are wired into that agent's config. -//! 2. `blue run ` with a tool-forcing prompt — asserts exit 0, the -//! `mcp-started` + `mcp-called` component markers, and `BLUE_MCP_OK` in the -//! output (the marker the managed MCP tool returns). This proves the agent -//! really launched, the MCP server started and was called, and a real model -//! round-trip completed. +//! 2. `blue run ` with a tool-forcing prompt — asserts exit 0 and +//! `BLUE_MCP_OK` in the output (the marker the managed MCP tool returns). +//! Harnesses that preserve the marker-directory environment must also +//! create the `mcp-started` + `mcp-called` component markers; Codex must +//! instead emit a completed call for the exact managed server and tool in +//! its structured JSON event stream. //! 3. LiteLLM spend logs — asserts this test's user (every test runs as its //! own unique user) produced an inference that resolved to the governed //! upstream model, and that the recorded credential is a hashed virtual @@ -45,6 +46,42 @@ use std::time::Duration; use e2e_slim::{AgentMatrix, AgentSelection, Home}; +fn value_contains_string(value: &serde_json::Value, needle: &str) -> bool { + match value { + serde_json::Value::String(value) => value.contains(needle), + serde_json::Value::Array(values) => values + .iter() + .any(|value| value_contains_string(value, needle)), + serde_json::Value::Object(values) => values + .values() + .any(|value| value_contains_string(value, needle)), + _ => false, + } +} + +fn codex_has_completed_certify_call(output: &str) -> bool { + output.lines().any(|line| { + let Ok(event) = serde_json::from_str::(line) else { + return false; + }; + event.get("type").and_then(serde_json::Value::as_str) == Some("item.completed") + && event.pointer("/item/type").and_then(serde_json::Value::as_str) + == Some("mcp_tool_call") + && event.pointer("/item/server").and_then(serde_json::Value::as_str) + == Some("e2e-remote") + && event.pointer("/item/tool").and_then(serde_json::Value::as_str) + == Some("blue_certify") + && event.pointer("/item/status").and_then(serde_json::Value::as_str) + == Some("completed") + && event + .pointer("/item/error") + .is_none_or(serde_json::Value::is_null) + && event + .pointer("/item/result") + .is_some_and(|result| value_contains_string(result, "BLUE_MCP_OK")) + }) +} + /// The current lock-pin version for `agent` (the newest/blessed cell). Read from /// the shared lock so it tracks the single source of truth for the pin. fn lock_pin_version(agent: &str) -> String { @@ -220,14 +257,21 @@ fn certify_cell(agent: &str, version: &str) { output.status.success(), "`blue run {agent}` exited non-zero:\n{combined}" ); - assert!( - markers.join("mcp-started").is_file(), - "{agent}: mcp-started marker missing (MCP server never launched):\n{combined}" - ); - assert!( - markers.join("mcp-called").is_file(), - "{agent}: mcp-called marker missing (managed MCP tool never invoked):\n{combined}" - ); + if agent == "codex" { + assert!( + codex_has_completed_certify_call(&combined), + "{agent}: output missing a completed e2e-remote/blue_certify MCP call:\n{combined}" + ); + } else { + assert!( + markers.join("mcp-started").is_file(), + "{agent}: mcp-started marker missing (MCP server never launched):\n{combined}" + ); + assert!( + markers.join("mcp-called").is_file(), + "{agent}: mcp-called marker missing (managed MCP tool never invoked):\n{combined}" + ); + } assert!( combined.contains("BLUE_MCP_OK"), "{agent}: output missing the BLUE_MCP_OK MCP marker:\n{combined}" @@ -275,3 +319,35 @@ fn certify_cell(agent: &str, version: &str) { // including the lock-pin cell per agent (which preserves today's per-agent // certification coverage). include!(concat!(env!("OUT_DIR"), "/matrix_cert_cases.rs")); + +#[cfg(test)] +mod tests { + use super::codex_has_completed_certify_call; + + #[test] + fn accepts_completed_codex_certify_call_with_missing_or_null_error() { + for output in [ + concat!( + "not json\n", + r#"{"type":"item.completed","item":{"type":"mcp_tool_call","server":"e2e-remote","tool":"blue_certify","status":"completed","result":{"content":[{"type":"text","text":"BLUE_MCP_OK"}]}}}"#, + ), + r#"{"type":"item.completed","item":{"type":"mcp_tool_call","server":"e2e-remote","tool":"blue_certify","status":"completed","result":{"content":[{"type":"text","text":"BLUE_MCP_OK"}]},"error":null}}"#, + ] { + assert!(codex_has_completed_certify_call(output), "rejected: {output}"); + } + } + + #[test] + fn rejects_non_evidence_codex_events() { + for output in [ + r#"{"type":"item.started","item":{"type":"mcp_tool_call","server":"e2e-remote","tool":"blue_certify","status":"in_progress","result":"BLUE_MCP_OK"}}"#, + r#"{"type":"item.completed","item":{"type":"mcp_tool_call","server":"other","tool":"blue_certify","status":"completed","result":"BLUE_MCP_OK"}}"#, + r#"{"type":"item.completed","item":{"type":"mcp_tool_call","server":"e2e-remote","tool":"other","status":"completed","result":"BLUE_MCP_OK"}}"#, + r#"{"type":"item.completed","item":{"type":"mcp_tool_call","server":"e2e-remote","tool":"blue_certify","status":"failed","result":"BLUE_MCP_OK"}}"#, + r#"{"type":"item.completed","item":{"type":"mcp_tool_call","server":"e2e-remote","tool":"blue_certify","status":"completed","result":"BLUE_MCP_OK","error":{"message":"tool failed"}}}"#, + r#"{"type":"item.completed","item":{"type":"agent_message","text":"BLUE_MCP_OK"}}"#, + ] { + assert!(!codex_has_completed_certify_call(output), "accepted: {output}"); + } + } +} diff --git a/tests/e2e/specs/journey.spec.ts b/tests/e2e/specs/journey.spec.ts index 184aafd..1917eba 100644 --- a/tests/e2e/specs/journey.spec.ts +++ b/tests/e2e/specs/journey.spec.ts @@ -196,6 +196,7 @@ test.describe.serial("Blue deployment journey", () => { expect(originalResponse.status(), await originalResponse.text()).toBe(200); const original = await originalResponse.json(); const gatewayConfig = YAML.parse(original.managed_yaml); + gatewayConfig.harnesses.codex.managed_config.model = "gpt-e2e"; gatewayConfig.gateway = { type: "litellm" }; const gatewayResponse = await page.request.put(`${control}/admin/governance-config`, { data: { @@ -1587,11 +1588,11 @@ esac ); await memberPage.getByRole("button", { name: "Provision key" }).click(); expect((await posted).status()).toBe(200); - await expect(memberPage.getByRole("alert")).toContainText("gateway account is not provisioned: member has no upstream account"); + await expect(memberPage.getByText("gateway account is not provisioned: member has no upstream account", { exact: true })).toBeVisible(); await expect(memberPage.getByRole("tab", { name: "Key" })).toBeVisible(); await memberPage.reload(); - await expect(memberPage.getByRole("alert")).toContainText("gateway account is not provisioned: member has no upstream account"); + await expect(memberPage.getByText("gateway account is not provisioned: member has no upstream account", { exact: true })).toBeVisible(); const access = await memberContext.request.get(`${control}/gateway/key`); expect(access.status(), await access.text()).toBe(200); expect(await access.json()).toMatchObject({ status: "error", error: "gateway account is not provisioned: member has no upstream account" }); diff --git a/tests/e2e/specs/zz-dashboard-filtering.spec.ts b/tests/e2e/specs/zz-dashboard-filtering.spec.ts index dc1fd3f..1fb4c62 100644 --- a/tests/e2e/specs/zz-dashboard-filtering.spec.ts +++ b/tests/e2e/specs/zz-dashboard-filtering.spec.ts @@ -842,7 +842,7 @@ test.describe.serial("dashboard table filtering", () => { await dialog.getByRole("button", { name: "Regenerate link" }).click(); await expect(dialog.getByRole("heading", { name: "Share replacement invitation link" })).toBeVisible(); - const replacementUrl = await dialog.getByLabel("Invitation link").inputValue(); + const replacementUrl = await dialog.getByRole("textbox", { name: "Invitation link" }).inputValue(); const replacementId = new URL(replacementUrl).searchParams.get("id"); expect(replacementId).toBeTruthy(); expect(replacementId).not.toBe(original.id);