fix(ci): stabilize cross-platform e2e checks

.github/workflows/e2e-windows.yml

@@ -37,6 +37,8 @@ jobs:
           try { & ./tests/e2e-windows/run.ps1 } finally { Stop-Transcript }
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         if: always()
+        # The native journey above is the gate; diagnostics must not override its result.
+        continue-on-error: true
         with:
           name: windows-cli-e2e
           path: tests/e2e-windows/artifacts

tests/e2e-slim/README.md

@@ -32,7 +32,7 @@ mutable or retired Docker Hub tags.
 | `all_agents_config` | The **same** model + MCP + skill assertions for `codex` / `kimi` / `opencode`, so all four agents are covered identically. | yes (per agent) | no |
 | `session_upload` | `blue session-upload` per harness → detached worker presigns → PUTs to MinIO → completes; the list shows all 4, and one artifact is downloaded and byte-compared to the upload. | no | no |
 | `agent_matrix_config` | **Tier A (version matrix).** The same model + MCP + skill assertions as `all_agents_config`, but per `(agent, version)` cell across multiple CLI versions (not just the pin). No inference. Self-skips unless the matrix is enabled. | yes (per cell) | no |
-| `agent_certification` | **Tier B (version matrix).** Per `(agent, version)` cell, one uniform body: `blue run <agent>` launches the real agent through the real gateway, forces the managed `blue_certify` MCP tool (asserts `mcp-started` + `mcp-called` markers and `BLUE_MCP_OK`), and asserts a LiteLLM spend log recorded the governed model alias with a swapped virtual key rather than the client inference JWT. With the matrix off it certifies just the pin per agent (today's coverage). | yes (per cell) | **yes** (opt-in) |
+| `agent_certification` | **Tier B (version matrix).** Per `(agent, version)` cell, one uniform body: `blue run <agent>` launches the real agent through the real gateway and forces the managed `blue_certify` MCP tool. Codex certification inspects its structured JSONL event stream for a completed `e2e-remote/blue_certify` call containing `BLUE_MCP_OK`; Claude, Kimi, and OpenCode assert the `mcp-started` + `mcp-called` marker files and `BLUE_MCP_OK`. The test also asserts that a LiteLLM spend log recorded the governed model alias with a swapped virtual key rather than the client inference JWT. With the matrix off it certifies just the pin per agent (today's coverage). | yes (per cell) | **yes** (opt-in) |
 
 **Real vs. faked.** Real: control-api, Postgres, MinIO (real presigned PUT/GET),
 the `blue` CLI, the agent CLIs (really installed, version-probed for eligibility),

tests/e2e-slim/tests/agent_certification.rs

@@ -9,11 +9,12 @@
 //! Each `<agent>_certifies` test:
 //!   1. `blue apply --yes` — asserts the managed `e2e-remote` MCP server and the
 //!      managed `example` skill are wired into that agent's config.
-//!   2. `blue run <agent>` with a tool-forcing prompt — asserts exit 0, the
-//!      `mcp-started` + `mcp-called` component markers, and `BLUE_MCP_OK` in the
-//!      output (the marker the managed MCP tool returns). This proves the agent
-//!      really launched, the MCP server started and was called, and a real model
-//!      round-trip completed.
+//!   2. `blue run <agent>` with a tool-forcing prompt — asserts exit 0 and
+//!      `BLUE_MCP_OK` in the output (the marker the managed MCP tool returns).
+//!      Harnesses that preserve the marker-directory environment must also
+//!      create the `mcp-started` + `mcp-called` component markers; Codex must
+//!      instead emit a completed call for the exact managed server and tool in
+//!      its structured JSON event stream.
 //!   3. LiteLLM spend logs — asserts this test's user (every test runs as its
 //!      own unique user) produced an inference that resolved to the governed
 //!      upstream model, and that the recorded credential is a hashed virtual
@@ -45,6 +46,42 @@ use std::time::Duration;
 
 use e2e_slim::{AgentMatrix, AgentSelection, Home};
 
+fn value_contains_string(value: &serde_json::Value, needle: &str) -> bool {
+    match value {
+        serde_json::Value::String(value) => value.contains(needle),
+        serde_json::Value::Array(values) => values
+            .iter()
+            .any(|value| value_contains_string(value, needle)),
+        serde_json::Value::Object(values) => values
+            .values()
+            .any(|value| value_contains_string(value, needle)),
+        _ => false,
+    }
+}
+
+fn codex_has_completed_certify_call(output: &str) -> bool {
+    output.lines().any(|line| {
+        let Ok(event) = serde_json::from_str::<serde_json::Value>(line) else {
+            return false;
+        };
+        event.get("type").and_then(serde_json::Value::as_str) == Some("item.completed")
+            && event.pointer("/item/type").and_then(serde_json::Value::as_str)
+                == Some("mcp_tool_call")
+            && event.pointer("/item/server").and_then(serde_json::Value::as_str)
+                == Some("e2e-remote")
+            && event.pointer("/item/tool").and_then(serde_json::Value::as_str)
+                == Some("blue_certify")
+            && event.pointer("/item/status").and_then(serde_json::Value::as_str)
+                == Some("completed")
+            && event
+                .pointer("/item/error")
+                .is_none_or(serde_json::Value::is_null)
+            && event
+                .pointer("/item/result")
+                .is_some_and(|result| value_contains_string(result, "BLUE_MCP_OK"))
+    })
+}
+
 /// The current lock-pin version for `agent` (the newest/blessed cell). Read from
 /// the shared lock so it tracks the single source of truth for the pin.
 fn lock_pin_version(agent: &str) -> String {
@@ -220,14 +257,21 @@ fn certify_cell(agent: &str, version: &str) {
         output.status.success(),
         "`blue run {agent}` exited non-zero:\n{combined}"
     );
-    assert!(
-        markers.join("mcp-started").is_file(),
-        "{agent}: mcp-started marker missing (MCP server never launched):\n{combined}"
-    );
-    assert!(
-        markers.join("mcp-called").is_file(),
-        "{agent}: mcp-called marker missing (managed MCP tool never invoked):\n{combined}"
-    );
+    if agent == "codex" {
+        assert!(
+            codex_has_completed_certify_call(&combined),
+            "{agent}: output missing a completed e2e-remote/blue_certify MCP call:\n{combined}"
+        );
+    } else {
+        assert!(
+            markers.join("mcp-started").is_file(),
+            "{agent}: mcp-started marker missing (MCP server never launched):\n{combined}"
+        );
+        assert!(
+            markers.join("mcp-called").is_file(),
+            "{agent}: mcp-called marker missing (managed MCP tool never invoked):\n{combined}"
+        );
+    }
     assert!(
         combined.contains("BLUE_MCP_OK"),
         "{agent}: output missing the BLUE_MCP_OK MCP marker:\n{combined}"
@@ -275,3 +319,35 @@ fn certify_cell(agent: &str, version: &str) {
 // including the lock-pin cell per agent (which preserves today's per-agent
 // certification coverage).
 include!(concat!(env!("OUT_DIR"), "/matrix_cert_cases.rs"));
+
+#[cfg(test)]
+mod tests {
+    use super::codex_has_completed_certify_call;
+
+    #[test]
+    fn accepts_completed_codex_certify_call_with_missing_or_null_error() {
+        for output in [
+            concat!(
+                "not json\n",
+                r#"{"type":"item.completed","item":{"type":"mcp_tool_call","server":"e2e-remote","tool":"blue_certify","status":"completed","result":{"content":[{"type":"text","text":"BLUE_MCP_OK"}]}}}"#,
+            ),
+            r#"{"type":"item.completed","item":{"type":"mcp_tool_call","server":"e2e-remote","tool":"blue_certify","status":"completed","result":{"content":[{"type":"text","text":"BLUE_MCP_OK"}]},"error":null}}"#,
+        ] {
+            assert!(codex_has_completed_certify_call(output), "rejected: {output}");
+        }
+    }
+
+    #[test]
+    fn rejects_non_evidence_codex_events() {
+        for output in [
+            r#"{"type":"item.started","item":{"type":"mcp_tool_call","server":"e2e-remote","tool":"blue_certify","status":"in_progress","result":"BLUE_MCP_OK"}}"#,
+            r#"{"type":"item.completed","item":{"type":"mcp_tool_call","server":"other","tool":"blue_certify","status":"completed","result":"BLUE_MCP_OK"}}"#,
+            r#"{"type":"item.completed","item":{"type":"mcp_tool_call","server":"e2e-remote","tool":"other","status":"completed","result":"BLUE_MCP_OK"}}"#,
+            r#"{"type":"item.completed","item":{"type":"mcp_tool_call","server":"e2e-remote","tool":"blue_certify","status":"failed","result":"BLUE_MCP_OK"}}"#,
+            r#"{"type":"item.completed","item":{"type":"mcp_tool_call","server":"e2e-remote","tool":"blue_certify","status":"completed","result":"BLUE_MCP_OK","error":{"message":"tool failed"}}}"#,
+            r#"{"type":"item.completed","item":{"type":"agent_message","text":"BLUE_MCP_OK"}}"#,
+        ] {
+            assert!(!codex_has_completed_certify_call(output), "accepted: {output}");
+        }
+    }
+}

tests/e2e/specs/journey.spec.ts

@@ -196,6 +196,7 @@ test.describe.serial("Blue deployment journey", () => {
     expect(originalResponse.status(), await originalResponse.text()).toBe(200);
     const original = await originalResponse.json();
     const gatewayConfig = YAML.parse(original.managed_yaml);
+    gatewayConfig.harnesses.codex.managed_config.model = "gpt-e2e";
     gatewayConfig.gateway = { type: "litellm" };
     const gatewayResponse = await page.request.put(`${control}/admin/governance-config`, {
       data: {
@@ -1587,11 +1588,11 @@ esac
       );
       await memberPage.getByRole("button", { name: "Provision key" }).click();
       expect((await posted).status()).toBe(200);
-      await expect(memberPage.getByRole("alert")).toContainText("gateway account is not provisioned: member has no upstream account");
+      await expect(memberPage.getByText("gateway account is not provisioned: member has no upstream account", { exact: true })).toBeVisible();
       await expect(memberPage.getByRole("tab", { name: "Key" })).toBeVisible();
 
       await memberPage.reload();
-      await expect(memberPage.getByRole("alert")).toContainText("gateway account is not provisioned: member has no upstream account");
+      await expect(memberPage.getByText("gateway account is not provisioned: member has no upstream account", { exact: true })).toBeVisible();
       const access = await memberContext.request.get(`${control}/gateway/key`);
       expect(access.status(), await access.text()).toBe(200);
       expect(await access.json()).toMatchObject({ status: "error", error: "gateway account is not provisioned: member has no upstream account" });

tests/e2e/specs/zz-dashboard-filtering.spec.ts

@@ -842,7 +842,7 @@ test.describe.serial("dashboard table filtering", () => {
     await dialog.getByRole("button", { name: "Regenerate link" }).click();
     await expect(dialog.getByRole("heading", { name: "Share replacement invitation link" })).toBeVisible();
 
-    const replacementUrl = await dialog.getByLabel("Invitation link").inputValue();
+    const replacementUrl = await dialog.getByRole("textbox", { name: "Invitation link" }).inputValue();
     const replacementId = new URL(replacementUrl).searchParams.get("id");
     expect(replacementId).toBeTruthy();
     expect(replacementId).not.toBe(original.id);